diff --git a/cmd/mesh-controller/accept_outside.go b/cmd/mesh-controller/accept_outside.go index 2489f542..dd83125e 100644 --- a/cmd/mesh-controller/accept_outside.go +++ b/cmd/mesh-controller/accept_outside.go @@ -112,11 +112,13 @@ func acceptForProvision(ctx context.Context, provider, providerModule, provision if len(accepted.Sealed) == 0 { return nil } - if err := sendTo(ctx, open, accepted.Machines); err != nil { - return fmt.Errorf("%s is sealed and could not be sent: %w; `push %s` sends it", provision, err, - strings.Join(accepted.Machines, "`, `push ")) + // Composed into each consumer's composition (novox/hq ADR 0539 clause 5): a new revision its want then names. + if _, err := composeModules(ctx, open, accepted.Machines); err != nil { + return fmt.Errorf("%s is sealed and its compositions on %s could not be written: %w", provision, + strings.Join(accepted.Machines, ", "), err) } - fmt.Printf(" sent %s; each consumer starts again on it\n", strings.Join(accepted.Machines, ", ")) + fmt.Printf(" composed for %s; each consumer starts again on it once its want names the new composition\n", + strings.Join(accepted.Machines, ", ")) return nil } diff --git a/cmd/mesh-controller/acts.go b/cmd/mesh-controller/acts.go index 9a8cd694..3215b1aa 100644 --- a/cmd/mesh-controller/acts.go +++ b/cmd/mesh-controller/acts.go @@ -58,13 +58,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio // Every line of the answer comes from the one resolution after the write (novox/hq issue 237): // what the reads before it would say about a machine left out is not the answer's. ctx = leftOutSaidBy(ctx, io.Discard) - // Held while it is recorded, so it cannot land between a converge's preview and its flip and - // be taken without ever having been previewed (novox/hq ADR 0100). - ctx, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() // **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue // 325): a build in flight keeps the assignment pending, a module known and not built is said with how to // build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act @@ -171,8 +164,21 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio answer += "\n but " + catalogue.WrongMachine(u.Module, c, node) } } - return answer + fmt.Sprintf("\n run `push %s` to send it", node) + - blockedElsewhere(ctx, open, node), nil + return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil +} + +// composedAfterAct composes the node an assignment or its end changed (novox/hq ADR 0539 clause 5): what the +// answer says of it. The assigner then writes the node's wants by its rules; nothing is sent. +func composedAfterAct(ctx context.Context, open *stores, node string) string { + wrote, err := composeModules(ctx, open, []string{node}) + if err != nil { + return fmt.Sprintf("\n but its compositions on %s could not all be written: %v", node, firstLine(err.Error())) + } + if len(wrote[node]) == 0 { + return fmt.Sprintf("\n %s's compositions were already as composed now", node) + } + return fmt.Sprintf("\n composed for %s (%d composition(s) written): its wants follow by the assigner's rules", node, + len(wrote[node])) } // seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the @@ -231,11 +237,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string) } // One resolution after the write, as assign's answer (novox/hq issue 237). ctx = leftOutSaidBy(ctx, io.Discard) - ctx, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() shelf, err := open.inventory.Catalogue(ctx) if err != nil { return "", err @@ -273,8 +274,8 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string) return "", err } } - answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so", - node, strings.Join(modules, ", "), node) + answer := fmt.Sprintf("%s no longer runs %s: its want is written removed by the assigner", + node, strings.Join(modules, ", ")) after := resolvedAfter(ctx, open, shelf, node) for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) { answer += "\n " + line @@ -295,7 +296,7 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string) for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) { answer += "\n " + line } - return answer + blockedElsewhere(ctx, open, node), nil + return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil } // afterAct is the one resolution of a node an act's answer is said from, taken after its write diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index 5a5191c1..9a95be52 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -6,6 +6,10 @@ import ( "context" "encoding/json" "errors" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" "net/http" "reflect" "strings" @@ -50,6 +54,7 @@ func anAdoptedAnchor(t *testing.T) (*stores, *[]string) { t.Fatal(err) } } + onTheBusWithItsHaves(t, open) sent := &[]string{} saved := sendNodes sendNodes = func(_ context.Context, _ *stores, names []string) error { @@ -83,13 +88,6 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held .. if err != nil { t.Fatal(err) } - record, err := open.inventory.NodeByName(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil { - t.Fatal(err) - } if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{ Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body), Firewall: "ufw", Held: held, Reachable: reachable, @@ -178,7 +176,7 @@ func TestTakingNamesWhatItReplaces(t *testing.T) { t.Fatal(err) } if !strings.Contains(said, "container hello-web (hello-web.server)") || - !strings.Contains(said, "push anchor") { + !strings.Contains(said, "compositions of anchor are written") { t.Fatalf("taking did not say what it replaces and what to run:\n%s", said) } } @@ -250,6 +248,7 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { t.Fatal(err) } + *sent = nil // the take composed the node; what follows is judged from here reportsHolding(t, open, heldFile) preview, err := converge(ctx, open, "anchor", false, "", "") @@ -438,6 +437,7 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) { if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { t.Fatal(err) } + *sent = nil // the take composed the node; what follows is judged from here reportsHolding(t, open, heldFile) preview, err := converge(ctx, open, "anchor", false, "", "") if err != nil { @@ -495,52 +495,6 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) { } } -// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is -// composed after it — never sent after it with the node still adopted. And the send inside the -// flip is not made to wait on the flip's own hold. -func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) { - open, _ := anAdoptedAnchor(t) - ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - reportsHolding(t, open, heldFile) - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - var heldElsewhere, heldHere error - sendNodes = func(inner context.Context, open *stores, names []string) error { - // Another caller cannot hold the node while the flip sends it. - waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond) - defer cancel() - if release, err := open.inventory.HoldNodes(waiting, names); err == nil { - release() - heldElsewhere = errors.New("another caller held the node while the flip sent it") - } - // The flip's own send holds it without waiting on itself. - _, release, err := holdNodes(inner, open, names) - if err != nil { - heldHere = err - return err - } - release() - return nil - } - if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { - t.Fatal(err) - } - if heldElsewhere != nil || heldHere != nil { - t.Fatalf("%v %v", heldElsewhere, heldHere) - } - // And given back once it is done. - release, err := open.inventory.HoldNodes(ctx, []string{"anchor"}) - if err != nil { - t.Fatal(err) - } - release() -} - // novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found, // not only its files, and the digest changes when any of them does. func TestThePreviewNamesEveryHeldKind(t *testing.T) { @@ -592,6 +546,7 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) { if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { t.Fatal(err) } + *sent = nil // the take composed the node; what follows is judged from here // Only a loopback listener: nothing off the machine, which is the same silence. reportsReaching(t, open, []link.Reach{ {Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"}, @@ -612,32 +567,94 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) { } } -// An assignment cannot land between a preview and the flip that takes every module: assigning -// holds the node, so it waits for whatever is converging it. -func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) { - open, _ := anAdoptedAnchor(t) +// onTheBusWithItsHaves gives the controller a bus that holds, for the anchor, a want and a have of its module that agree: the node +// has everything wanted of it, as read from the haves (novox/hq ADR 0539 §9), so a flip is refused for what it is about and +// not because the bus could not be read. +func onTheBusWithItsHaves(t *testing.T, open *stores) { + t.Helper() ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - reportsHolding(t, open, heldFile) - preview, err := converge(ctx, open, "anchor", false, "", "") + js, err := broker.Dial(testbus.URL(t)) if err != nil { t.Fatal(err) } - saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll - inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond - defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }() - - var whileFlipping error - sendNodes = func(context.Context, *stores, []string) error { - _, whileFlipping = assign(ctx, open, "anchor", "notes") - return nil - } - if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { + t.Cleanup(js.Close) + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + api, err := jetstream.New(js.Conn()) + if err != nil { t.Fatal(err) } - if !errors.Is(whileFlipping, inventory.ErrNodeBusy) { - t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping) + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64) + composed := "sha256:" + strings.Repeat("b", 64) + put := func(bucket, key string, value []byte) { + kv, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket}) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + } + wantKey, err := sdkdelivery.WantKey("anchor", "hello-web") + if err != nil { + t.Fatal(err) + } + haveKey, err := sdkdelivery.HaveKey("anchor", "hello-web") + if err != nil { + t.Fatal(err) + } + want, err := sdkdelivery.EncodeWant(sdkdelivery.Want{FP: fp, Composed: composed}) + if err != nil { + t.Fatal(err) + } + have, err := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: fp, Composed: composed, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}) + if err != nil { + t.Fatal(err) + } + put(sdkdelivery.AssignerWantsBucket, wantKey, want) + put(sdkdelivery.HaveBucket, haveKey, have) +} + +// **When the bus cannot be read, whether the machine runs what is wanted is not known, and the flip says exactly that** (issue +// 1945): not "push", and not "has not reported". +func TestTheFlipIsRefusedWithTheBusUnreachableSayingItCannotTell(t *testing.T) { + open, sent := anAdoptedAnchor(t) + reportsHolding(t, open, heldContainer, heldFile) + servingBus.Store(nil) // this process has no bus now, and no address to dial + _, err := converge(t.Context(), open, "anchor", true, "", "") + if err == nil || !strings.Contains(err.Error(), "cannot tell whether anchor runs what is wanted: the bus is not reachable") || + strings.Contains(err.Error(), "push") { + t.Fatalf("the flip with the bus out of reach: %v", err) + } + if len(*sent) != 0 { + t.Fatal("a refused flip sent something") + } +} + +// **A take composes the node only when it is performed** (issue 1945): a preview, a take that names no digest and a take refused +// for a changed preview compose nothing; the performed take composes the machine once, and says so. +func TestATakeComposesTheNodeOnlyWhenItIsPerformed(t *testing.T) { + open, sent := anAdoptedAnchor(t) + ctx := t.Context() + reportsHolding(t, open, heldContainer, heldFile) + preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{}) + if err != nil { + t.Fatal(err) + } + if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil { + t.Fatal("a take of a held module without the preview's digest was performed") + } + if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: "not-the-digest"}); err == nil { + t.Fatal("a take naming another preview's digest was performed") + } + if len(*sent) != 0 { + t.Fatalf("a preview or a refused take composed %v", *sent) + } + said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: takeDigestIn(t, preview)}) + if err != nil { + t.Fatal(err) + } + if len(*sent) != 1 || (*sent)[0] != "anchor" || !strings.Contains(said, "compositions of anchor are written") { + t.Fatalf("the performed take composed %v and said:\n%s", *sent, said) } } diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 0ff12403..b61a124d 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -205,9 +205,9 @@ func adoptedNodes(nodes []inventory.Node) []string { // The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the // command API alike, so a refusal is the same refusal in the same words at both (ADR 0035). -// sendNodes sends the named machines what they should be now. A variable so a test can see what -// an act would send without a broker. -var sendNodes = sendTo +// sendNodes composes the named machines as they should be now (novox/hq ADR 0539 clause 5): each node's want then +// names what changed. A variable so a test can see what an act would compose without a broker. +var sendNodes = composeAndSay // DefaultFilter is the module converging a node assigns to load the mesh's derived filter. const DefaultFilter = "nftables" @@ -276,8 +276,8 @@ func take(ctx context.Context, open *stores, node, module string, opts takeOptio // is still the machine: the same two refusals the flip makes. if age := time.Since(c.reported.At); age > reportFreshFor { return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+ - "an account newer than %s: run `push %s --wait 2m`, then preview again", - node, age.Round(time.Second), reportFreshFor, node) + "an account newer than %s: wait for its next report, then preview again", + node, age.Round(time.Second), reportFreshFor) } if opts.Digest == "" { return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+ @@ -294,9 +294,12 @@ func take(ctx context.Context, open *stores, node, module string, opts takeOptio } said := fmt.Sprintf("%s is taken on %s", module, node) if holds { - said += "; the next push replaces what the node found and holds for it:\n" + preview + said += "; its next composition replaces what the node found and holds for it:\n" + preview } - return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil + if err := sendNodes(ctx, open, []string{node}); err != nil { + return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err + } + return said + fmt.Sprintf("\n the compositions of %s are written; its wants then cut it over", node), nil } // comparison is everything a take puts beside what the module declares: the machine's account of @@ -433,7 +436,7 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", ")) } if c.settingsRefused != "" { - fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: %s\n", c.settingsRefused) + fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the composition leaves it out: %s\n", c.settingsRefused) } preview = strings.TrimRight(b.String(), "\n") sum := sha256.Sum256([]byte(preview)) @@ -608,16 +611,6 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s if filter == "" { filter = DefaultFilter } - if yes { - // Held from the checks to the send, so no push composed before the flip is sent after it - // and returns the node to adopted. - held, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() - ctx = held - } record, err := inv.NodeByName(ctx, node) if err != nil { return "", err @@ -625,24 +618,27 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s if !record.Adopted { return "", fmt.Errorf("%s is converged already; there is nothing to flip", node) } - reports, err := inv.LastReports(ctx) + reports, err := lastReports(ctx, inv) if err != nil { return "", err } - current := false + current, unknown := false, false for _, r := range reports { if r.Node == node { - current = r.Current + current, unknown = r.Current, r.CurrentUnknown } } + if unknown { + return "", fmt.Errorf("cannot tell whether %s runs what is wanted: the bus is not reachable", node) + } reported, err := inv.AdoptionOf(ctx, node) if err != nil { return "", err } if !current || reported.At.IsZero() { - return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+ - "says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+ - "converge once it has applied", node, node) + return "", fmt.Errorf("%s does not run everything that is wanted of it yet, or has not reported, so what it "+ + "says is reachable may not be the machine as it is: converge once its haves match its wants and it has "+ + "reported", node) } assignedWhenPreviewed, err := inv.Assigned(ctx, node) @@ -738,13 +734,13 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s if yes && countReachable(reported) == 0 { return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+ "up ever is: its account looks partial — whatever reads what is listening, or what "+ - "the container runtime publishes, did not answer. Fix that on the machine and run "+ - "`push %s --wait 2m`, then preview again", node, node) + "the container runtime publishes, did not answer. Fix that on the machine, wait for its next report, "+ + "then preview again", node) } if age := time.Since(reported.At); age > reportFreshFor { return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+ - "only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+ - "then preview again", node, age.Round(time.Second), reportFreshFor, node) + "only on an account newer than %s: wait for its next report, then preview again", node, + age.Round(time.Second), reportFreshFor) } if digest == "" { return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+ @@ -787,7 +783,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s said += "; took " + strings.Join(took, ", ") } if err := sendNodes(ctx, open, []string{node}); err != nil { - return said + "\n and it could not be sent: run `push " + node + "`", err + return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err } return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil } @@ -1031,18 +1027,12 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) { if record.Adopted { return "", fmt.Errorf("%s is adopted already", node) } - held, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() - ctx = held if err := inv.SetAdopted(ctx, node, true); err != nil { return "", err } said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node) if err := sendNodes(ctx, open, []string{node}); err != nil { - return said + "\n and it could not be sent: run `push " + node + "`", err + return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err } return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil } diff --git a/cmd/mesh-controller/ask_bus_wait.go b/cmd/mesh-controller/ask_bus_wait.go deleted file mode 100644 index 4488e903..00000000 --- a/cmd/mesh-controller/ask_bus_wait.go +++ /dev/null @@ -1,52 +0,0 @@ -package main - -import ( - "fmt" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// busWaitBound is how long a module whose ask the bus did not answer is asked again at every look before the plan fails -// (issue 1334). -const busWaitBound = 10 * time.Minute - -// busWaiting says an open plan has a module of its current tier that the bus did not answer an ask of: the look must come back -// for it, as nothing else will when no build of the tier is in flight (issue 1334). -func busWaiting(p inventory.Plan) bool { - return len(busWaitingIn(p)) > 0 -} - -// busWaitingIn are the modules of the plan's current tier left unasked because the bus did not answer, sorted. -func busWaitingIn(p inventory.Plan) []string { - if !p.Open() || p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.State == "" && s.BusWaitFrom != nil { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// busWaitWords is how a verb says modules are waiting for the bus: not asked yet, and asked again by the plan at its next look. -func busWaitWords(p inventory.Plan, modules []string) string { - return fmt.Sprintf("%s not asked yet: the bus did not answer; %s asks again at its next look", strings.Join(modules, ", "), p.ID) -} - -// slicesWithout is names without those in drop, in order. -func slicesWithout(names, drop []string) []string { - var out []string - for _, n := range names { - if !slices.Contains(drop, n) { - out = append(out, n) - } - } - return out -} diff --git a/cmd/mesh-controller/ask_bus_wait_test.go b/cmd/mesh-controller/ask_bus_wait_test.go deleted file mode 100644 index 3df88f5a..00000000 --- a/cmd/mesh-controller/ask_bus_wait_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting: -// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue -// 1403, met again once the hand rebuild no longer resumed the plan first). -func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) { - since := time.Now().UTC() - p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, - Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}} - waiting := busWaitingOnRetry(p) - _, said := retryWords(p, []string{"a"}, nil, waiting) - if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") { - t.Fatalf("a retry left for the bus is said as %q", said) - } -} diff --git a/cmd/mesh-controller/asker_retry_integration_test.go b/cmd/mesh-controller/asker_retry_integration_test.go deleted file mode 100644 index 2ea3177f..00000000 --- a/cmd/mesh-controller/asker_retry_integration_test.go +++ /dev/null @@ -1,138 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own -// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1). -func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Delivery: &inventory.PlanDelivery{}, - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: "killed by hand"}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - // The controller's seat answers `plans` as it does: by running the verb. - sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) { - var args map[string]string - _ = json.Unmarshal(m.Data, &args) - answer := link.Answer{Result: json.RawMessage(`"ok"`)} - if len(args) != 1 || args["retry"] == "" { - answer = link.Answer{Error: "the retry was asked with more than the plan"} - } else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil { - answer = link.Answer{Error: err.Error()} - } - body, _ := json.Marshal(answer) - _ = m.Respond(body) - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - - // What S23 raises for it, read from the store as the watchdog reads it. - raised := func(context.Context) ([]conditions.Condition, error) { - facts, err := gatherFailedPlans(ctx, open.inventory, time.Now()) - if err != nil { - return nil, err - } - var out []conditions.Condition - for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) { - out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}) - } - return out, nil - } - now := time.Now() - store := memAskedStore{} - var sent []published - var acts []link.HandAct - a := &asker{open: raised, store: store, - silence: func(context.Context, string, time.Duration, string, string) error { return nil }, - publish: func(_ context.Context, subject string, body []byte, id string) error { - sent = append(sent, published{subject, id, body}) - return nil - }, - call: callAction(conn), - record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil }, - now: func() time.Time { return now }, - logf: t.Logf} - cs, _ := raised(ctx) - if len(cs) != 1 || len(cs[0].Actions) != 1 { - t.Fatalf("the failed plan raised %+v", cs) - } - if err := a.reconcile(ctx); err != nil { - t.Fatal(err) - } - var w asks.Warrant - for _, r := range store { - for _, o := range r.Ask.Options { - if o.Label == "Retry" { - w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen, - Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now, - AskDigest: r.Ask.Digest(), - By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} - } - } - } - if w.Ask == "" { - t.Fatalf("no ask offers Retry: %+v", store) - } - body, _ := json.Marshal(w) - if err := a.Decided(ctx, body); err != nil { - t.Fatal(err) - } - if err := a.Decided(ctx, body); err != nil { // heard again: nothing more - t.Fatal(err) - } - - p, err := open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - m := p.Modules["a"] - if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" { - t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m) - } - if got := store[w.Ask]; got.Acted != "done" { - t.Fatalf("the ask ended %q", got.Acted) - } - if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) { - t.Fatalf("the warrant was recorded as %+v", acts) - } - if cs, _ = raised(ctx); len(cs) != 0 { - t.Fatalf("the plan-failed condition did not end: %+v", cs) - } -} diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 40604d7e..4e732bee 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -125,8 +125,15 @@ func newAskerRig(t *testing.T) *askerRig { } func heldCondition() conditions.Condition { - o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", - Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0] + release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove, + Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}} + stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove, + Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}} + o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096", Kind: "delivery-stalled", + Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Headline: "Delivery of hq held for 36 hours", + Explanation: "A delivery of hq has been held for 36 hours, past its limit.", + Needs: "release it, or stop it, " + FromMeshMCPServer, Actions: []conditions.Action{release, stop}} return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline, Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} } @@ -299,35 +306,6 @@ func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) { } } -func TestEachAnswerCallsExactlyItsVerb(t *testing.T) { - plan := "plan-1791454185265004861" - waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent, - Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.", - Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)} - module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning, - Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.", - Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart", - Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove, - Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}} - for _, tc := range []struct { - c conditions.Condition - label string - want string - }{ - {waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`}, - {waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`}, - {module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`}, - } { - r := newAskerRig(t) - r.open = []conditions.Condition{tc.c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label)) - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) { - t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want) - } - } -} - func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{unitsCondition()} @@ -1026,215 +1004,3 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) { t.Errorf("held %v (%q, %v) with nothing to read", held, why, err) } } - -// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it. -func failedPlanCondition(plan string) conditions.Condition { - p := aFailedPlan(plan, nil) - f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)) - f.failed = []failedPlanFacts{failedFactsFor(p)} - o := watchPlansFailed(f)[0] - return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} -} - -// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again". -func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) { - r := newAskerRig(t) - c := failedPlanCondition("plan-1792433168676870614") - r.open = []conditions.Condition{c} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 1 || len(sent[0].Options) != 1 { - t.Fatalf("asked %+v", sent) - } - o := sent[0].Options[0] - if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" { - t.Errorf("the option is %+v", o) - } - if err := sent[0].Check(r.now); err != nil { - t.Errorf("the ask is refused as made: %v", err) - } - if strings.Contains(sent[0].Explanation, "mesh MCP server") { - t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation) - } -} - -// 1315/AC2: the warrant calls plans with retry= and nothing else, once, and is recorded as the hand act; -// the same warrant heard again does nothing. -func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) { - plan := "plan-1792433168676870614" - r := newAskerRig(t) - c := failedPlanCondition(plan) - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, c.Key, "Retry") - answerWith(t, r, w) - answerWith(t, r, w) - if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want { - t.Fatalf("called %v, want exactly %s", r.called, want) - } - if len(r.acts) != 1 { - t.Fatalf("hand-acts %+v", r.acts) - } - act := r.acts[0] - if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan || - act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer { - t.Errorf("the hand-act is %+v", act) - } - if got := r.store[w.Ask]; got.Acted != "done" { - t.Errorf("kept %+v", got) - } -} - -// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is -// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before. -func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) { - plan := "plan-1792433168676870614" - waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent, - Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.", - Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)} - for _, tc := range []struct { - name string - c conditions.Condition - label string - }{ - {"retry", failedPlanCondition(plan), "Retry"}, - {"stop", waiting, "Stop"}, - } { - t.Run(tc.name, func(t *testing.T) { - r := newAskerRig(t) - r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"} - r.open = []conditions.Condition{tc.c} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, tc.c.Key, tc.label) - answerWith(t, r, w) - answerWith(t, r, w) // heard again: told once - if len(r.told) != 1 { - t.Fatalf("told %+v", r.told) - } - got := r.told[0] - if got.headline != tc.c.Headline { - t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline) - } - said := strings.Join(got.lines, " ") - if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) { - t.Errorf("the lines do not carry the answer and the refusal's words: %q", said) - } - if why, ok := conditions.Plain(said); !ok { - t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said) - } - if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") { - t.Errorf("the failure is not recorded as before: %+v", r.acts) - } - if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") { - t.Errorf("kept %+v", kept) - } - }) - } -} - -// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here. -func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) { - r := newAskerRig(t) - c := failedPlanCondition("plan-1792433168676870614") - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) - if len(r.told) != 0 { - t.Fatalf("a retry that was done told %+v", r.told) - } - - r = newAskerRig(t) - r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"} - module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning, - Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.", - Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart", - Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove, - Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}} - r.open = []conditions.Condition{module} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, module.Key, "Restart")) - if len(r.told) != 0 || len(r.acts) != 1 { - t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts) - } -} - -// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was -// changed"; and the sentence a refusal falls back to fits the answer given. -func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) { - plan := "plan-1792433168676870614" - r := newAskerRig(t) - r.refuses = errors.New("mesh-controller did not answer plans within 30s") - c := failedPlanCondition(plan) - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) - if len(r.told) != 1 { - t.Fatalf("told %+v", r.told) - } - said := strings.Join(r.told[0].lines, " ") - if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") { - t.Errorf("a call that may have run is told as %q", said) - } - if why, ok := conditions.Plain(said); !ok { - t.Errorf("not plain: %s: %q", why, said) - } -} - -func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) { - plan := "plan-1792433168676870614" - for _, tc := range []struct { - act conditions.Action - want string - }{ - {retryAction(plan), "a retry would not run now"}, - {conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"}, - {conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"}, - } { - // Words the operator's channel withholds: a hash and a dotted name. - got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act) - if !strings.Contains(got, tc.want) { - t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want) - } - if _, ok := conditions.Plain(got); !ok { - t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got) - } - } -} - -// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the -// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not. -func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) { - r := newAskerRig(t) - c := failedPlanCondition("plan-1792433168676870614") - c.Raised = r.now.Add(-time.Minute) - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) - // Still open right after the answer: not asked again at once. - r.now = r.now.Add(time.Minute) - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != 1 { - t.Fatalf("asked again while the same condition stayed open: %d", n) - } - // The retry ran, the plan left failed and the condition ended. - r.now = r.now.Add(time.Minute) - r.open = nil - _ = r.a.reconcile(context.Background()) - // It fails again, ten minutes after the answer: a condition raised after the answered ask ended. - r.now = r.now.Add(8 * time.Minute) - again := failedPlanCondition("plan-1792433168676870614") - again.Raised = r.now - r.open = []conditions.Condition{again} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - if n := len(r.asksSent(t)); n != 2 { - t.Fatalf("the second failure was asked %d times in all, want 2", n) - } - if len(r.called) != 1 { - t.Fatalf("something retried without an answer: %v", r.called) - } -} diff --git a/cmd/mesh-controller/awaiting_push_integration_test.go b/cmd/mesh-controller/awaiting_push_integration_test.go deleted file mode 100644 index b18d766f..00000000 --- a/cmd/mesh-controller/awaiting_push_integration_test.go +++ /dev/null @@ -1,186 +0,0 @@ -//go:build integration - -package main - -import ( - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/compose" - "git.novox.be/novox/mesh-sdk/go/overlay" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - - // The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing - // can be compared about a secret that does not exist (status), and nothing here string-matches. - _, _, err := composeQuestion(ctx, open, "laptop", compose.Reading, false) - var notMade *catalogue.NotMadeError - if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" { - t.Fatalf("a read composition did not say which secret is not made, typed: %v", err) - } - // Foreseen, it composes, names what the push makes, and made nothing. - foreseen, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false) - if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" { - t.Fatalf("foreseen: %v %v", foreseen.foreseen, err) - } - if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held { - t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err) - } - - // Within the bound: nothing at all — no urgent, no warning, nobody notified. - if got := d1(t, open); len(got) != 0 { - t.Fatalf("a machine waiting for a push raised %+v", got) - } - - // Past the bound: a warning, not urgent, saying what the push will make. - before := awaitingPushBound - awaitingPushBound = -time.Minute - t.Cleanup(func() { awaitingPushBound = before }) - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning || - !strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") { - t.Fatalf("a machine left un-pushed past the bound: %+v", got) - } - - // Pushed: the secret is made and D1 says nothing. - pushedBy(t, open, "laptop") - if got := d1(t, open); len(got) != 0 { - t.Fatalf("a pushed machine still raised %+v", got) - } -} - -func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aTalker()) - if _, err := assign(ctx, open, "laptop", "talker"); err != nil { - t.Fatal(err) - } - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent || - !strings.Contains(got[0].Summary, "module issue talker --node laptop") { - t.Fatalf("a push that will be refused was not said urgently: %+v", got) - } - - // And a machine whose composition fails for anything else, with a secret waiting beside it, is - // uncomposable for that — the stand-in hides nothing. - register(t, open, aKeeper()) - if _, err := assign(ctx, open, "anchor", "keeper"); err != nil { - t.Fatal(err) - } - one, two := rivals() - register(t, open, one) - register(t, open, two) - _, _ = assign(ctx, open, "anchor", "rival-one") - _, _ = assign(ctx, open, "anchor", "rival-two") - keys := map[string]conditions.Severity{} - for _, o := range d1(t, open) { - keys[o.Key()] = o.Severity - } - if keys["machine.anchor.uncomposable"] != conditions.Urgent { - t.Fatalf("a real failure beside a waiting secret: %v", keys) - } -} - -func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil { - t.Fatal(err) - } - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") { - t.Fatalf("a given secret under an old key: %+v", got) - } -} - -func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") - sent := time.Now() - since, err := open.inventory.AwaitingSince(ctx, "laptop") - if err != nil || since.After(sent) { - t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent) - } - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - since, err = open.inventory.AwaitingSince(ctx, "laptop") - if err != nil || since.Before(sent.Add(-time.Second)) { - t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err) - } -} - -func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - // What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only - // when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown. - if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil { - t.Fatal(err) - } - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - sends, err := readDeliveries(ctx, open.inventory) - if err != nil { - t.Fatal(err) - } - if sends["laptop"].settled("keeper", time.Now()) { - t.Fatal("a holder assigned and not pushed is expected to answer") - } - if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) { - t.Fatal("a module the machine was sent long ago is not expected to answer") - } - pushedBy(t, open, "laptop") - sends, err = readDeliveries(ctx, open.inventory) - if err != nil { - t.Fatal(err) - } - if sends["laptop"].settled("keeper", time.Now()) { - t.Fatal("pushed a moment ago and not reported, a holder is already expected") - } - if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) { - t.Fatal("pushed past the grace, a holder is not expected") - } - if _, ok := sends["laptop"].carried["keeper"]; !ok { - t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried) - } - pushedAndApplied(t, open, "laptop") - if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) { - t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err) - } -} diff --git a/cmd/mesh-controller/awaiting_push_test.go b/cmd/mesh-controller/awaiting_push_test.go deleted file mode 100644 index 74b7097b..00000000 --- a/cmd/mesh-controller/awaiting_push_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package main - -import ( - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed -// the machine's declaration without making anything, failed on the missing secret, and raised an urgent -// "nothing can be sent to " — seen live on 2026-10-06, a desktop notification for a machine -// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a -// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning. - -// aKeeper is a module with an own secret the mesh makes — a backup repository's password. -func aKeeper() catalogue.Manifest { - return catalogue.Manifest{Module: "keeper", Version: "1", - OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}}, - Resources: []map[string]any{ - {"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"}, - }} -} - -// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets -// are made — without a bus to carry it. -func pushedBy(t *testing.T, open *stores, node string) string { - t.Helper() - ctx := t.Context() - plan, settings, err := resolveNode(ctx, open, node) - if err != nil { - t.Fatal(err) - } - declared, err := declarationFor(ctx, open, node, plan, settings) - if err != nil { - t.Fatal(err) - } - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - record, err := open.inventory.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - digest := digestOf(body) - // A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was - // built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109). - builds := declared.Builds - if builds == nil { - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - builds = map[string]string{} - for _, m := range plan.Modules { - builds[m.Module] = current[m.Module].Commit - } - } - if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil { - t.Fatal(err) - } - return digest -} - -// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration. -func pushedAndApplied(t *testing.T, open *stores, node string) { - t.Helper() - digest := pushedBy(t, open, node) - record, err := open.inventory.NodeByName(t.Context(), node) - if err != nil { - t.Fatal(err) - } - if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{ - Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil { - t.Fatal(err) - } -} - -// d1 runs D1 once. -func d1(t *testing.T, open *stores) []conditions.Observation { - t.Helper() - got, err := probeDeclarations(t.Context(), &doctor{open: open}) - if err != nil { - t.Fatal(err) - } - return got -} - -// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is -// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing. - -// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make. -// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words. - -// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is -// refused on it, so D1 is too. - -// The bound is read from when the machine began waiting: the oldest assignment since its last send. - -// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent -// it and had time to report — never between assign and push. -func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) { - now := time.Now() - sent := now.Add(-time.Minute) - long := now.Add(-time.Hour) - cases := []struct { - name string - send lastSend - want bool - }{ - {"never sent", lastSend{}, false}, - {"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false}, - {"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false}, - {"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true}, - {"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true}, - {"sent before builds were kept", lastSend{sent: &long, current: true}, true}, - } - for _, c := range cases { - if got := c.send.settled("keeper", now); got != c.want { - t.Errorf("%s: settled %v, want %v", c.name, got, c.want) - } - } -} - -// And through the stores: assigned, not in the last send; pushed and reported, carried and settled. diff --git a/cmd/mesh-controller/away.go b/cmd/mesh-controller/away.go deleted file mode 100644 index fa04575a..00000000 --- a/cmd/mesh-controller/away.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the -// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send -// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent -// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so. - -// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine -// is away once presence says so). A machine -// never heard from has not joined and is not away. -func awayOf(machines []machineFacts, now time.Time) map[string]bool { - out := map[string]bool{} - for _, m := range machines { - if m.silent(now) { - out[m.name] = true - } - } - return out -} - -// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and -// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did -// before: a machine is not passed on a reading that was not made. -var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) { - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return nil, err - } - address, err := broker.BusAddress() - if err != nil { - return nil, err - } - js, err := broker.Dial(address) - if err != nil { - return nil, err - } - defer js.Close() - machines := make([]machineFacts, len(nodes)) - for i, n := range nodes { - machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen} - } - if err := withPresence(ctx, js.Conn(), machines, now); err != nil { - return nil, err - } - return awayOf(machines, now), nil -} - -// rememberAway keeps on the plan the machines of running that are away: sorted, each once. -func rememberAway(p *inventory.Plan, running []string, away map[string]bool) { - for _, n := range running { - if away[n] && !slices.Contains(p.Away, n) { - p.Away = append(p.Away, n) - } - } - sort.Strings(p.Away) -} - -// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and -// those it did not pass as away. -func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string { - return slices.DeleteFunc(slices.Clone(running), func(n string) bool { - if !slices.Contains(p.Away, n) { - return false - } - _, sent := s.Rest[n] - return !sent - }) -} - -// awaySaying is what a plan says of the machines it passed as away, empty when it passed none. -func awaySaying(p inventory.Plan) string { - var said []string - for _, n := range p.Away { - said = append(said, n+" was away: sent when it is heard again") - } - return strings.Join(said, "; ") -} - -// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot. -func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool { - away, err := judgeAway(ctx, open, now) - if err != nil { - fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err) - return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above - } - return away -} diff --git a/cmd/mesh-controller/away_integration_test.go b/cmd/mesh-controller/away_integration_test.go deleted file mode 100644 index 45156cdd..00000000 --- a/cmd/mesh-controller/away_integration_test.go +++ /dev/null @@ -1,263 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it -// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard -// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's -// builds, so the send it gets when it is heard again carries them (AC3). -func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - // laptop is away as every tier is sent; anchor is heard. - wasAway := judgeAway - judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) { - return map[string]bool{"laptop": true}, nil - } - t.Cleanup(func() { judgeAway = wasAway }) - - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - sentHeard := make(chan *nats.Msg, 32) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - wasSaved := inventory.PlanSaved - inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - - // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. - var mu sync.Mutex - var sends [][]string - everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it - // Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look - // goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask - // was where the walk paused). - wholeKeys := openPlanKeys - var holdB atomic.Bool - holdB.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{all, "b", &holdB, ""}, done, err - } - digests := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - mu.Lock() - defer mu.Unlock() - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range []string{"a", "b"} { - builds[m] = current[m].Commit - } - for _, node := range names { - digests++ - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - register := func(m, commit string, asked time.Time) { - t.Helper() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - record := func(id, m, commit string, at time.Time) { - t.Helper() - b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - old := time.Now().Add(-2 * time.Hour) - for _, m := range []string{"a", "b"} { - record("build-"+m+"-1", m, "c1", old) - register(m, "c1", old) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { - t.Fatal(err) - } - } - record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) - register("a", "c2", time.Now().Add(-time.Minute)) - - // A machine's report on what it was sent, heard as the controller hears it. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - seq := int64(0) - reports := func(node string) { - t.Helper() - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - t.Fatalf("%s was sent nothing: %v", node, err) - } - seq++ - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": seq}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - plan := func() inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, "plan-1160") - if err != nil { - t.Fatal(err) - } - return p - } - - built := time.Now().UTC() - p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(), - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, - "b": {}, - }} - first, stopFirst := context.WithCancel(ctx) - defer stopFirst() - waitFirst := startRolloutEvents(first, open, js) - // Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338). - t.Cleanup(func() { stopFirst(); waitFirst() }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - eventually(t, "tier 0 sent to anchor alone", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor" - }) - reports("anchor") - // The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own. - var moved string - eventually(t, "tier 1 reached on anchor's report alone", func() bool { - if q := plan(); q.Tier == 1 { - moved = q.Note - return true - } - return false - }) - if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") { - t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved) - } - // Tier 1's module is taken from its key once that is built, which the walk's look does by itself. - holdB.Store(false) - askToLook() - eventually(t, "tier 1 sent to anchor alone", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor" - }) - // 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again. - was, known, err := inv.SentBuilds(ctx, "laptop") - if err != nil || !known { - t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - if was["a"] == current["a"].Commit || was["a"] != "c1" { - t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit) - } - if got := plan().Away; strings.Join(got, ",") != "laptop" { - t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got) - } -} - -// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of -// the fingerprint fp when that is given. -type keysHolding struct { - planKeyReads - module string - held *atomic.Bool - fp string -} - -func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) { - if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) { - return builderkeys.Key{Module: module, Fingerprint: fp}, nil - } - return k.planKeyReads.Build(ctx, module, fp) -} - -// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared, -// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of -// the test, and the flag is set until the test lowers it. -func holdKeys(t *testing.T, modules ...string) *atomic.Bool { - t.Helper() - held := &atomic.Bool{} - held.Store(true) - was := openPlanKeys - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := was() - var keys planKeyReads = all - for _, m := range modules { - keys = keysHolding{keys, m, held, ""} - } - return keys, done, err - } - t.Cleanup(func() { openPlanKeys = was }) - return held -} diff --git a/cmd/mesh-controller/away_test.go b/cmd/mesh-controller/away_test.go deleted file mode 100644 index f31e9a48..00000000 --- a/cmd/mesh-controller/away_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the -// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so. - -// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away. -func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) { - now := time.Now().UTC() - machines := []machineFacts{ - {name: "heard", lastHeard: now.Add(-time.Minute)}, - {name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)}, - {name: "stuck", lastHeard: now, presence: presenceSilent}, - {name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard}, - {name: "new"}, - } - got := awayOf(machines, now) - for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false, - "new": false} { - if got[node] != want { - t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got) - } - } -} - -// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once. -func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) { - p := &inventory.Plan{} - away := map[string]bool{"g14": true, "ace": true} - rememberAway(p, []string{"anchor", "g14"}, away) - rememberAway(p, []string{"g14", "ace"}, away) - if !slices.Equal(p.Away, []string{"ace", "g14"}) { - t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away) - } -} - -// 1161/AC2: the plan says it, where its other passed machines are said. -func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) { - say := awaySaying(inventory.Plan{Away: []string{"g14"}}) - if !strings.Contains(say, "g14 was away: sent when it is heard again") { - t.Fatalf("1161/AC2: the plan says %q", say) - } - if got := awaySaying(inventory.Plan{}); got != "" { - t.Fatalf("1161/AC2: a plan that passed no machine says %q", got) - } -} - -// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for. -func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) { - p := inventory.Plan{Away: []string{"g14"}} - none := &inventory.PlanModule{} - if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) { - t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got) - } - sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}} - if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) { - t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got) - } -} diff --git a/cmd/mesh-controller/backups_before_send_integration_test.go b/cmd/mesh-controller/backups_before_send_integration_test.go deleted file mode 100644 index 33e30492..00000000 --- a/cmd/mesh-controller/backups_before_send_integration_test.go +++ /dev/null @@ -1,312 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "os" - "strings" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and -// answers `backed-up` with the last night it is told (novox/hq ADR 0404). -type aBackupSeatOnTheBus struct { - mu sync.Mutex - nows []string - night map[string]any // module → lastNight -} - -func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) { - t.Helper() - reply := func(m *nats.Msg, result any) { - body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node}) - _ = m.Respond(body) - } - now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) { - var a struct { - Module string `json:"module"` - } - _ = json.Unmarshal(m.Data, &a) - f.mu.Lock() - f.nows = append(f.nows, a.Module) - f.mu.Unlock() - reply(m, map[string]any{"started": a.Module}) - }) - if err != nil { - t.Fatal(err) - } - read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) { - f.mu.Lock() - defer f.mu.Unlock() - var out []map[string]any - for module, n := range f.night { - out = append(out, map[string]any{"module": module, "lastNight": n}) - } - reply(m, out) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() }) - if err := conn.Flush(); err != nil { - t.Fatal(err) - } -} - -func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) { - f.mu.Lock() - defer f.mu.Unlock() - f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}} -} - -func (f *aBackupSeatOnTheBus) asks() int { - f.mu.Lock() - defer f.mu.Unlock() - return len(f.nows) -} - -// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits -// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the -// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the -// backups again, and the plan goes on. One bus and one store for the scenarios. -func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) - notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` + - `{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` + - `"resources":[{"id":"files","type":"directory"}]}`)) - if err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - if _, err := assign(ctx, open, node, "keeper"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, node, "notes"); err != nil { - t.Fatal(err) - } - } - seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}} - for node, s := range seats { - s.serve(t, js.Conn(), node) - } - // Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup. - wasWithin, wasEvery := backupsWithin, backupsPollEvery - backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond - t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery }) - - newPlan := func(id string) { - t.Helper() - p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(), - State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - } - read := func(id string) inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, id) - if err != nil { - t.Fatal(err) - } - return p - } - - // 1. Both machines are asked; while one has not answered, no build is asked. - newPlan("plan-one") - advancePlans(ctx, open) - if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 { - t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked) - } - seats["anchor"].answers("notes", true, "") - advancePlans(ctx, open) - if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending { - t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups) - } - seats["laptop"].answers("notes", true, "") - advancePlans(ctx, open) - if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone { - t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups) - } - finish := read("plan-one") - finish.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &finish); err != nil { - t.Fatal(err) - } - - // 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised. - seats["anchor"].night, seats["laptop"].night = nil, nil - newPlan("plan-two") - advancePlans(ctx, open) - seats["anchor"].answers("notes", true, "") - seats["laptop"].answers("notes", false, "no space left on device") - advancePlans(ctx, open) - held := read("plan-two") - if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 { - t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked) - } - var failed *inventory.PlanBackup - for i := range held.Backups.Asked { - if held.Backups.Asked[i].State == inventory.BackupFailed { - failed = &held.Backups.Asked[i] - } - } - if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" || - strings.Contains(held.Note, "tried again") { - t.Fatalf("the failed backup: %+v note=%q", failed, held.Note) - } - advancePlans(ctx, open) // not retried on the next look - if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 || - seats["laptop"].asks() != 2 { - t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks()) - } - said, err := probeBackupHeld(ctx, &doctor{open: open}) - if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") { - t.Fatalf("the condition: %+v %v", said, err) - } - - // 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on. - seats["laptop"].answers("notes", false, "no space left on device") - if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by", - "mesh-delivery for jochen"}); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil { - t.Fatal("a plan with no failed backup was asked again") - } - seats["laptop"].night = nil - advancePlans(ctx, open) - if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 { - t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks()) - } - seats["laptop"].answers("notes", true, "") - advancePlans(ctx, open) - // One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key - // build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes - // it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key. - if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 || - got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" { - t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked) - } - if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 { - t.Fatalf("the condition stays: %+v", said) - } -} - -// Issue 1086: a plan cut from a merge asked its first tier at the cut, so a module keeping data was sent without the -// backup the plan promised. The cut now takes the backups first (askFirstTierAtCut, batches.go), and the walk's own -// step asks the tier once they are done. -func TestAPlanCutFromAMergeOfABackedUpModuleAsksItsBackupAndNotItsTierUntilItIsDone(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - everyHeadWaits(t) - asked := asksRecorded(t) - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` + - `{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` + - `"resources":[{"id":"files","type":"directory"}]}`)) - if err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/notes", Seat: "git", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, m := range []string{"keeper", "notes"} { - if _, err := assign(ctx, open, "anchor", m); err != nil { - t.Fatal(err) - } - } - seat := &aBackupSeatOnTheBus{} - seat.serve(t, js.Conn(), "anchor") - wasWithin, wasEvery := backupsWithin, backupsPollEvery - backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond - t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery }) - - // 1. A merge of notes is heard and its window closes: the batch is cut through cutBatch. - hear(t, open, repoMerge("notes", "c-notes", t0), t0.Add(time.Second)) - cutAt(t, open, t0.Add(time.Second+90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walk(s)", len(ws)) - } - // 2. Right after the cut no tier is asked, the record has anchor/notes pending, the seat heard one `now`. - cut, err := inv.PlanByID(ctx, ws[0].ID) - if err != nil { - t.Fatal(err) - } - if len(*asked) != 0 || askedInTier(cut) { - t.Fatalf("the first tier was asked at the cut: %v %+v", *asked, cut.Modules) - } - if cut.Backups == nil || cut.Backups.State != inventory.BackupPending || len(cut.Backups.Asked) != 1 || - cut.Backups.Asked[0].Machine != "anchor" || cut.Backups.Asked[0].Module != "notes" { - t.Fatalf("the backups of the cut: %+v", cut.Backups) - } - if seat.asks() != 1 { - t.Fatalf("the seat heard %d `now`", seat.asks()) - } - // 3. The backup answers done: the walk's own step asks the first tier, once, and asks no second backup. - seat.answers("notes", true, "") - advancePlans(ctx, open) - advancePlans(ctx, open) - after, err := inv.PlanByID(ctx, ws[0].ID) - if err != nil { - t.Fatal(err) - } - if len(*asked) != 1 || after.Backups.State != inventory.BackupDone || seat.asks() != 1 { - t.Fatalf("builds=%v backups=%+v asks=%d", *asked, after.Backups, seat.asks()) - } -} diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go deleted file mode 100644 index 28e63435..00000000 --- a/cmd/mesh-controller/batches.go +++ /dev/null @@ -1,1821 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "slices" - "sort" - "strconv" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **Merges are assembled in a rolling window, and each batch is delivered by one walk** (novox/hq ADR 0276, -// issue 362). -// -// Every merge opened a walk of its own, and the next merge of the branch superseded it. On 2026-10-10 two -// catalogue merges eighteen seconds apart left one walk that no delivery held, and the operator started it by -// hand 58 minutes later. A merge now joins the open **batch**. Each merge restarts the **merge window**; it -// closes when none came for `merge-window` (90 seconds) or at the latest `merge-window-at-most` (10 minutes) -// after the batch's first merge. A closed batch becomes one walk when no walk is open: one plan for every -// commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier -// ones. The walk names every merge it answers. -// -// - One walk at a time beside a related one (ADR 0407 decision 4, issue 973). Merges that come while a walk -// runs join the next batch; a closed batch that shares nothing with an open walk is cut beside it. A walk -// that started is never superseded; one still waiting for its delivery's word is folded into the walk of a -// batch related to it, which answers every merge it answered — the only supersession left. -// - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch -// still assembling, an open walk that builds everything it moves, or a walk done that did, which then -// names it at once. Otherwise it joins the next batch. -// - A failed walk marks nothing delivered. Each earlier merge it carried in a later commit is walked on its -// own commit, the newest first, one walk each, until one is delivered; the older ones are then answered -// by that one. -// -// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions -// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery -// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Such a batch is cut -// at once, beside any open walk it shares nothing with, and queued behind the first one it is related to -// (cutUnrelated). -// -// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with -// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds -// no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no -// catalogue batch's walk starts without the word (decided during the build, 2026-10-10). -// -// The batch, its merges and their times are in the store (batched_merge, and the batch's own plan record in -// the state `assembling` or `queued`), so a restarted controller resumes the window where it stood. - -// The merge window's defaults. The controller's settings `merge-window` and `merge-window-at-most` reach it in -// the file MESH_MERGE_WINDOW_FILE names (module.json), read at every look, so a change takes effect without a -// restart. -var ( - mergeWindowDefault = 90 * time.Second - mergeWindowAtMostDefault = 10 * time.Minute -) - -// mergeWindowFileVar names the file the controller's window settings are written to. -const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE" - -// batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before -// the controller is said to have failed to cut it (ADR 0276 decision 9). -const batchLateAfter = time.Minute - -// mergeGrace is how long before a batch's first merge a merge may have been made and still be told apart from one -// made before the controller began keeping merges (owedLate). -const mergeGrace = 10 * time.Minute - -// edgeChangeOrder is the order the rules of a plan draw among its modules (novox/hq ADR 0419 clauses 3 and 4, delivery_edges.go) -// made an edge of its tiers: the module after depends on the module before, and is gated on it. -const edgeChangeOrder = "change-order" - -// mergeWindowOf is the window and its maximum, from the controller's settings; a value unreadable is its -// default, and said. -func mergeWindowOf() (window, atMost time.Duration) { - window, atMost = mergeWindowDefault, mergeWindowAtMostDefault - path := os.Getenv(mergeWindowFileVar) - if path == "" { - return window, atMost - } - raw, err := os.ReadFile(path) - if err != nil { - fmt.Fprintf(os.Stderr, "the merge window's settings cannot be read (%v); %s and %s are used\n", err, window, atMost) - return window, atMost - } - var values map[string]any - if err := json.Unmarshal(raw, &values); err != nil { - fmt.Fprintf(os.Stderr, "the merge window's settings are not JSON (%v); %s and %s are used\n", err, window, atMost) - return window, atMost - } - for key, to := range map[string]*time.Duration{"merge-window": &window, "merge-window-at-most": &atMost} { - v, has := values[key] - if !has || v == "" { - continue - } - if d, ok := durationSetting(v); ok { - *to = d - } else { - fmt.Fprintf(os.Stderr, "the setting %s=%v is no duration; %s is used\n", key, v, *to) - } - } - return window, atMost -} - -// durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds. -func durationSetting(v any) (time.Duration, bool) { - switch t := v.(type) { - case string: - if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 { - return d, true - } - if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 { - return time.Duration(n * float64(time.Second)), true - } - case float64: - if t >= 0 { - return time.Duration(t * float64(time.Second)), true - } - } - return 0, false -} - -// mergeFacts is what hearMerge judges a merge by, kept apart so the catch-up (ADR 0457) judges a merge the way hearing it will. -type mergeFacts struct { - merged time.Time - touches []inventory.Entry - later []inventory.BatchedMerge - owed bool - from, packaging []inventory.Entry - already int - addsModuleToBuilds bool // a first merge into a repository the mesh builds nothing from: firstBuilds acts on it -} - -// factsOfMerge reads what hearing a merge turns on: the modules it touches whatever the history, the later merges of its -// branch, whether it is owed late, and the modules it moves. -func factsOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, - entries []inventory.Entry, read map[string][]inventory.ReadRepository) (mergeFacts, error) { - repository := m.Owner + "/" + m.Repo - f := mergeFacts{} - var err error - f.merged, err = time.Parse(time.RFC3339Nano, m.MergedAt) - if err != nil { - f.merged = now - } - // **What it touches, history or not** (review of this change): a merge heard after a later merge of its - // branch was cut reads as history for every module that cut marked seen, and was dropped from the record - // here, never named by the walk that carried it. - raw := m - raw.MergedAt = "" - f.touches = wouldMove(raw, entries, read) - if len(f.touches) > 0 { - if f.later, err = inv.LaterMergesOf(ctx, repository, m.Base, f.merged.UTC()); err != nil { - return f, err - } - } - if f.owed, err = owedLate(ctx, inv, m, f.merged, f.touches); err != nil { - return f, err - } - f.from, f.packaging, f.already = mergeCandidates(m, entries, read) - f.addsModuleToBuilds = f.already == 0 && len(addedDirs(m, entries, read)) > 0 - return f, nil -} - -// plans says hearing the merge puts it in a batch or a walk. -func (f mergeFacts) plans() bool { - return len(f.from) > 0 || len(f.packaging) > 0 || len(f.later) > 0 || f.owed -} - -// heldMerge marks a failure to act on a merge, whatever it was, as "not now": the announcement is held on the bus and -// handed over again, so the merge joins the next batch, never taken (issue 1583, after 921). The bus acknowledges it -// only when hearMerge answers without ErrTryAgain, and a merge heard once is not announced again: the bus answering -// "jetstream not enabled", a bucket missing for a moment or a store read cut short each lost the merge when only an -// outage of a known kind (notNow) was held. The delay between hands-over is bounded (link.MergeRedeliverAfter). -func heldMerge(err error) error { - if err == nil { - return nil - } - if errors.Is(err, link.ErrTryAgain) { - return err - } - return fmt.Errorf("%w: the merge is not kept yet: %w", link.ErrTryAgain, err) -} - -// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard, -// and cuts what is due. -func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error { - // One merge acted on at a time, whoever hands it over (novox/hq issue 266). - actingOnMerges.Lock() - defer actingOnMerges.Unlock() - - inv := f.open.inventory - repository := m.Owner + "/" + m.Repo - if kept, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil { - return heldMerge(err) - } else if known { - fmt.Printf("%s merged into %s (%.8s) was heard before; %s answers it\n", repository, m.Base, m.Commit, - orWords(kept.Plan, "a walk of its own commit, once the walks before it end")) - return nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return heldMerge(err) - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return heldMerge(err) - } - facts, err := factsOfMerge(ctx, inv, m, now, entries, read) - if err != nil { - return heldMerge(err) - } - merged, touches := facts.merged, facts.touches - if !facts.plans() { - already := facts.already - // "Already built from it" and "nothing reads it" are different facts, and reading the first - // as the second sends somebody looking for a broken trigger when the mesh is up to date. - if already > 0 { - fmt.Printf("%s merged into %s (%.8s); %d module(s) the mesh holds are already built from it\n", - repository, m.Base, m.Commit, already) - return nil - } - if already == 0 { - if done, err := f.firstBuilds(ctx, m, entries, read, now); err != nil || done { - return heldMerge(err) - } - } - fmt.Print(nothingReadsIt(repository, m)) - return nil - } - // A module the merge adds is registered from the builder's first done key of it, which the builder's write wakes - // (novox/hq ADR 0501 §6): nothing is asked here. - // **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk. - own := ownPath(touches) - waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries)) - if err != nil { - return heldMerge(err) - } - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return heldMerge(err) - } - defer release() - // Again under the hold: a controller handing over may have kept it since. - if _, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil || known { - return heldMerge(err) - } - var moves []string - for _, e := range touches { - moves = append(moves, e.Manifest.Module) - } - sort.Strings(moves) - event, err := json.Marshal(keptMerge{SourceMoved: m, Moves: moves}) - if err != nil { - return err - } - heard := inventory.BatchedMerge{Repository: repository, Branch: m.Base, Commit: m.Commit, Merged: merged.UTC(), - Heard: now, Event: event} - - // **A merge heard after a later merge of its branch** is answered by the walk holding that one. - // **A merge on the controller's own path never shares a batch with one that waits for the delivery's - // word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start - // with the controller's and skip their turn. Each kind has a batch of its own. - if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil { - return heldMerge(err) - } else if ok { - heard.Plan = p.ID - if _, err := inv.AddMerge(ctx, heard); err != nil { - return heldMerge(err) - } - if p.Batch() { - if err := keepBatch(ctx, inv, &p, now, ""); err != nil { - return unsaved(err) - } - } else if err := nameMerges(ctx, inv, &p); err != nil { - return unsaved(err) - } - fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+ - "answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p)) - return cutBatchesHeld(ctx, f.open, now) - } - - batch, why, err := batchToJoin(ctx, inv, heard, entries, read, now, own, waits) - if err != nil { - return heldMerge(err) - } - heard.Plan = batch.ID - if added, err := inv.AddMerge(ctx, heard); err != nil || !added { - return heldMerge(err) - } - holdFor(&batch, m.Head, waits, holding, own, now) - if err := keepBatch(ctx, inv, &batch, now, ""); err != nil { - return unsaved(err) - } - fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID, - batchWords(batch, now)) - return cutBatchesHeld(ctx, f.open, now) -} - -// holdFor keeps in a batch that a merge of a group joined whether that group still has a member to merge (ADR 0418 -// clause 3): held while it has, the window running again from the moment the last hold is cleared. A group with a -// merge of the controller's own path makes its batch an own-path one: such a plan waits for nobody's word, and is -// started by the controller when the group's batch closes (clause 6). -func holdFor(batch *inventory.Plan, head, why string, holding, own bool, now time.Time) { - if batch.Delivery == nil { - batch.Delivery = &inventory.PlanDelivery{} - } - if batch.Delivery.Batch == nil { - batch.Delivery.Batch = &inventory.PlanBatch{Own: own} - } - w := batch.Delivery.Batch - w.Own = w.Own || own - if head == "" { - return - } - if holding { - if w.Holds == nil { - w.Holds = map[string]string{} - } - w.Holds[head] = why - return - } - if _, held := w.Holds[head]; held { - holds := map[string]string{} - for h, y := range w.Holds { - if h != head { - holds[h] = y - } - } - w.Holds = holds - if len(holds) == 0 { - w.Holds, w.HeldAt = nil, now - } - } -} - -// owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut -// marks every module it moved seen at that moment, so a merge of that branch -// handed over late — made before the cut, heard after it — reads as history and was dropped. One that touches -// a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them, -// is kept; one made before it began is old news, as before. -func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time, - touches []inventory.Entry) (bool, error) { - if len(touches) == 0 { - return false, nil - } - kept, first, err := inv.KeptOnBranch(ctx, m.Owner+"/"+m.Repo, m.Base) - if err != nil || !kept { - return false, err - } - return !merged.Before(first.Add(-mergeGrace)), nil -} - -// answeredByALaterMerge is the plan that answers a merge heard after a later merge of its branch was put in -// one (ADR 0276 decision 4): a batch not yet cut, whose walk plans every file of the repository's merges; an -// open walk, or a walk done, that builds every module this merge would move — built from the branch after the -// later merge, which contains it. A failed or stopped walk answers nothing more, and a walk folded into -// another is followed to that one. False when none does: the merge joins the next batch. -func answeredByALaterMerge(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, - moves []inventory.Entry, own, held bool) (inventory.Plan, bool, error) { - later, err := inv.LaterMergesOf(ctx, heard.Repository, heard.Branch, heard.Merged) - if err != nil || len(later) == 0 { - return inventory.Plan{}, false, err - } - for _, l := range later { - p, err := followTakeOver(ctx, inv, l.Plan) - if err != nil { - return inventory.Plan{}, false, err - } - switch { - case p.Batch(): - // A batch of the other kind is not this merge's: it joins the batch of its own kind. - if p.OwnPath() == own { - return p, true, nil - } - case p.Open() || p.State == inventory.PlanDone: - // A walk that waited for nobody's word is not a catalogue merge's to be answered by while the delivery - // seat has a holder: its delivery would skip its turn (decided during the build, 2026-10-10). With no - // holder on record nothing waits, and any walk that built it answers. - waited := p.Delivery != nil && p.Delivery.Awaits != "" - if buildsEvery(p, moves) && (own || waited || !held) { - return p, true, nil - } - } - } - return inventory.Plan{}, false, nil -} - -// followTakeOver is a plan, or the walk that took it over when it was folded before it started. -func followTakeOver(ctx context.Context, inv *inventory.Inventory, id string) (inventory.Plan, error) { - p, err := inv.PlanByID(ctx, id) - for hops := 0; err == nil && hops < 16 && p.State == inventory.PlanSuperseded && p.Delivery != nil && - p.Delivery.TakenOverBy != ""; hops++ { - p, err = inv.PlanByID(ctx, p.Delivery.TakenOverBy) - } - return p, err -} - -// buildsEvery says a walk builds every one of the modules, and that there is one: a merge that moves nothing -// is no walk's to name. -func buildsEvery(p inventory.Plan, modules []inventory.Entry) bool { - if len(modules) == 0 { - return false - } - for _, e := range modules { - if _, in := p.Modules[e.Manifest.Module]; !in { - return false - } - } - return true -} - -// ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word. -func ownPath(touches []inventory.Entry) bool { - for _, e := range touches { - if onTheOwnPath(e.Manifest.Module, e.Manifest) { - return true - } - } - return false -} - -// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, as ADR 0306 -// decisions 2 and 3 read it). batchToJoin is the batch a merge heard now joins, and why in words: -// -// - an assembling or queued batch of its kind it is related to (related.go): it joins it and waits for the -// window, which its coming restarts. Related to two or more, the newer are folded into the oldest, which -// answers all their merges: a merge never waits in two windows, and what it shares with each walks in one. -// - none, and its delivery group has a member still to merge, or mesh-delivery could not say (groupWait, waits -// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say -// why. -// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at -// once means only that: with a walk open that it is related to the batch is queued behind it as any closed -// batch is (ADR 0276 decision 4, ADR 0407 decision 4); beside one it shares nothing with it is cut. -func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) { - edges, err := inv.Dependencies(ctx) - if err != nil { - return inventory.Plan{}, "", err - } - // **A group is one batch, whatever kinds its members are** (novox/hq ADR 0418 clause 6): a merge joins the batch - // of the other kind that holds a merge from its head. - if group, ok, err := batchOfGroup(ctx, inv, announcedOf(heard).Head, heard.Repository); err != nil { - return inventory.Plan{}, "", err - } else if ok && group.OwnPath() != own && group.Delivery != nil && group.Delivery.Batch != nil { - group.Delivery.Batch.CutAtOnce = false - return group, "its group " + announcedOf(heard).Head + " has a merge in " + group.ID + ", a batch of the other kind: " + - "a group is one batch", nil - } - joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges) - if err != nil { - return inventory.Plan{}, "", err - } - if len(joins) > 0 { - into := joins[0] - why := "related to " + into.ID + " (" + whys[0] + ")" - if len(joins) > 1 { - plans, moves, err := foldInto(ctx, inv, &into, joins[1:]) - if err != nil { - return inventory.Plan{}, "", err - } - if err := inv.SavePlansMoving(ctx, plans, moves); err != nil { - return inventory.Plan{}, "", err - } - var folded []string - for i, b := range joins[1:] { - folded = append(folded, b.ID+" ("+whys[i+1]+")") - } - why += ", and to " + strings.Join(folded, ", ") + ", folded into it" - } - into.Delivery.Batch.CutAtOnce = false - return into, why, nil - } - if waits != "" { - // Its own window, beside the batches it shares nothing with: waiting is for its group's members, which - // are related to it by their head and join it (ADR 0306 decision 2). - b, err := newBatch(ctx, inv, heard, now, own, false, waits) - return b, waits + ", it waits for the window in a batch of its own", err - } - b, err := newBatch(ctx, inv, heard, now, own, true, "") - return b, "it shares nothing with a batch assembling and no member of a group is still to merge: it is cut at once", err -} - -// relatedBatches are the batches not yet cut of a kind, oldest first, parted into those related to merges (with -// the first reason of each) and the others. Each batch is judged by what its cut would move (ADR 0306 decision -// 3): its merges' moves as they were kept and as they read now — the catalogue may have moved since a merge was -// heard — widened to their dependents, and, for a catalogue batch, every walk waiting for its word, which the -// cut folds in (cutBatch's carry). A merge kept with no moves leaves its batch unknown, related to everything. -// except is a walk not to count as folded: the one being stopped. -func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inventory.BatchedMerge, own bool, - except string, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge) (joins []inventory.Plan, whys []string, others []inventory.Plan, err error) { - batches, err := inv.Batches(ctx) - if err != nil { - return nil, nil, nil, err - } - var kind []inventory.Plan - for _, b := range batches { - if b.OwnPath() == own { - kind = append(kind, b) - } - } - if len(kind) == 0 { - return nil, nil, nil, nil - } - carry := newFootprint() - if !own { - // An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch folds those it is related to - // (cutUnrelated). Every one is counted here, which only makes a merge join a batch more readily: it waits - // for the window, never walks beside what it shares something with (issue 973). - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, nil, nil, err - } - for _, p := range plans { - if p.Release != nil || !p.Waiting() || p.ID == except { - continue - } - kept, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return nil, nil, nil, err - } - carry = unionFootprint(carry, footprintOfWalk(p, kept)) - } - } - mine := footprintAsCut(merges, entries, read, edges) - for _, b := range kind { - held, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return nil, nil, nil, err - } - if b.Delivery == nil { - b.Delivery = &inventory.PlanDelivery{} - } - if b.Delivery.Batch == nil { - b.Delivery.Batch = &inventory.PlanBatch{Own: own} - } - if ok, why := related(mine, unionFootprint(footprintAsCut(held, entries, read, edges), carry), edges); ok { - joins, whys = append(joins, b), append(whys, why) - } else { - others = append(others, b) - } - } - return joins, whys, others, nil -} - -// footprintAsCut is a batch's footprint as a cut now would read it: footprintOfBatch, widened by the modules each -// merge moves as the catalogue reads now (planBatch judges each merge again at the cut, so what it moves may -// have changed since it was heard). Both are kept: when in doubt, related. -func footprintAsCut(merges []inventory.BatchedMerge, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, edges []inventory.Edge) footprint { - f := footprintOfBatch(merges, edges) - var moved []string - for _, m := range merges { - s := announcedOf(m).SourceMoved - if s.Owner == "" { - continue - } - s.MergedAt = "" // news, whatever a cut marked seen since (as planBatch reads it) - for _, e := range wouldMove(s, entries, read) { - moved = append(moved, e.Manifest.Module) - } - } - for _, name := range reachableFrom(moved, edges) { - f.modules[name] = true - } - return f -} - -// unionFootprint is what two changes reach together. -func unionFootprint(a, b footprint) footprint { - f := newFootprint() - for _, from := range []footprint{a, b} { - for k := range from.repos { - f.repos[k] = true - } - for k := range from.modules { - f.modules[k] = true - } - for k := range from.heads { - f.heads[k] = true - } - } - f.unknown = a.unknown - if f.unknown == "" { - f.unknown = b.unknown - } - return f -} - -// foldInto folds batches into the oldest one related to a merge: the plans to write (into first, so no merge -// names a plan the store does not hold) and the merges to move, for one SavePlansMoving. The folded batches name -// the batch that took them over; a stop's closed window and its withheld modules (ADR 0299) go with their merges. -func foldInto(ctx context.Context, inv *inventory.Inventory, into *inventory.Plan, from []inventory.Plan) ([]*inventory.Plan, - []inventory.MergeMove, error) { - if into.Delivery == nil { - into.Delivery = &inventory.PlanDelivery{} - } - if into.Delivery.Batch == nil { - into.Delivery.Batch = &inventory.PlanBatch{Own: into.OwnPath()} - } - plans := []*inventory.Plan{into} - var moves []inventory.MergeMove - for i := range from { - f := from[i] - merges, err := inv.MergesOf(ctx, f.ID) - if err != nil { - return nil, nil, err - } - for _, m := range merges { - moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: into.ID, Alone: m.Alone}) - } - if f.Delivery == nil { - f.Delivery = &inventory.PlanDelivery{} - } - if w := f.Delivery.Batch; w != nil { - into.Delivery.Batch.Closed = into.Delivery.Batch.Closed || w.Closed - into.Delivery.Batch.Withheld = union(into.Delivery.Batch.Withheld, w.Withheld) - sort.Strings(into.Delivery.Batch.Withheld) - } - f.State = inventory.PlanSuperseded - f.Delivery.Batch = nil - f.Delivery.TakenOverBy = into.ID - f.Note = fmt.Sprintf("folded into %s before it was cut: a merge related to both joined it, and %s answers "+ - "its merges", into.ID, into.ID) - plans = append(plans, &f) - } - return plans, moves, nil -} - -// newBatch is a new batch of a kind, kept before its first merge names it: a merge never names a plan the store -// does not hold. cutAtOnce closes its window at once; waits says why it waits for its group, when it does. -func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own, - cutAtOnce bool, waits string) (inventory.Plan, error) { - b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, - Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling, - Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own, CutAtOnce: cutAtOnce, Waits: waits}}} - if err := inv.SavePlan(ctx, &b); err != nil { - return inventory.Plan{}, err - } - return b, nil -} - -// keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they -// answer, its window, and assembling or queued behind a walk. Saved only when that changed. -func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error { - return keepBatchBecause(ctx, inv, b, now, behind, "") -} - -// keepBatchBecause is keepBatch for a batch queued behind a walk it is related to, which its words say (issue 973). -func keepBatchBecause(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind, why string) error { - merges, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return err - } - window, atMost := mergeWindowOf() - // The note's seconds left are no change of the batch's: it is saved when what it holds or its state moved. - unNoted := func(p inventory.Plan) string { - p.Note = "" - return planSnapshot(p) - } - before := unNoted(*b) - commits, named := carriedOf(merges) - b.Commits = commits - if len(commits) > 0 { - newest := newestCommit(commits) - b.Repository, b.Branch, b.Commit, b.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged - } - if b.Delivery == nil { - b.Delivery = &inventory.PlanDelivery{} - } - anchor := b.Created - if was := b.Delivery.Batch; was != nil && was.HeldAt.After(anchor) { - anchor = was.HeldAt // the maximum counts from the last hold's end (ADR 0418 clause 3) - } - closes, latest := windowOf(merges, anchor, window, atMost) - b.Delivery.Merges = named - var closed, cutAtOnce bool - var withheld []string - var waits string - var holds map[string]string - var heldAt time.Time - if was := b.Delivery.Batch; was != nil { - holds, heldAt = was.Holds, was.HeldAt - closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299) - cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1) - waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2) - } - b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, BehindWhy: why, Own: b.OwnPath(), - Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits, Holds: holds, HeldAt: heldAt} - b.State = inventory.PlanAssembling - if behind != "" && windowClosed(*b, now) { - b.State = inventory.PlanQueued - } else { - b.Delivery.Batch.Behind, b.Delivery.Batch.BehindWhy = "", "" - } - b.Note = batchWords(*b, now) - if unNoted(*b) == before { - return nil - } - if err := inv.SavePlan(ctx, b); err != nil { - return err - } - // The window's clock follows the record, only when the record moved: a merge made the quiet key again, a - // batch that opened made its most (ADR 0407 decision 1). - armWindow(ctx, *b, now) - return nil -} - -// windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last -// merge heard plus the window, and the batch's first merge plus the maximum. -func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost time.Duration) (closes, latest time.Time) { - var last time.Time - for _, m := range merges { - if m.Heard.After(last) { - last = m.Heard - } - } - if last.IsZero() { - last = created - } - return last.Add(window), created.Add(atMost) -} - -// windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares -// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached. **A batch holding a group -// with a member still to merge is closed by none of the last three** (novox/hq ADR 0418 clause 3). -func windowClosed(b inventory.Plan, now time.Time) bool { - if b.Delivery == nil || b.Delivery.Batch == nil { - return true - } - w := b.Delivery.Batch - if w.Closed { - return true - } - if len(w.Holds) > 0 { - return false // a group's last member is still to merge: neither the window nor its maximum closes the batch - } - return w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost) -} - -// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and -// every merge named with the commit that carries it. -func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inventory.PlanMerge) { - latest := map[string]inventory.BatchedMerge{} - spelled := map[string]string{} - for _, m := range merges { - k := strings.ToLower(m.Repository) + "\x00" + m.Branch - if l, ok := latest[k]; !ok || laterMerge(m, l) { - latest[k] = m - } - var e link.SourceMoved - if json.Unmarshal(m.Event, &e) == nil && e.Owner != "" { - spelled[k] = e.Owner + "/" + e.Repo - } - } - var commits []inventory.PlanCommit - for k, l := range latest { - repo := spelled[k] - if repo == "" { - repo = l.Repository - } - commits = append(commits, inventory.PlanCommit{Repository: repo, Branch: l.Branch, Commit: l.Commit, Merged: l.Merged}) - } - sort.Slice(commits, func(i, j int) bool { - if !strings.EqualFold(commits[i].Repository, commits[j].Repository) { - return strings.ToLower(commits[i].Repository) < strings.ToLower(commits[j].Repository) - } - return commits[i].Branch < commits[j].Branch - }) - var named []inventory.PlanMerge - for _, m := range merges { - k := strings.ToLower(m.Repository) + "\x00" + m.Branch - repo := spelled[k] - if repo == "" { - repo = m.Repository - } - n := namedOf(m, repo) - if l := latest[k]; l.Commit != m.Commit { - n.Carried = l.Commit - } - named = append(named, n) - } - sort.SliceStable(named, func(i, j int) bool { - return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository) - }) - return commits, named -} - -// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was -// heard. The announcement reads back as a SourceMoved alone, which ignores the rest. -type keptMerge struct { - link.SourceMoved - Moves []string `json:"moves,omitempty"` - // FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first - // build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146). - FirstBuild bool `json:"first_build,omitempty"` - // Held says which first builds of the merge were refused for a word the running build-agent does not read: the merge - // waits for a build-agent that reads them, and is never failed (novox/hq ADR 0482 §5, issue 1173). - Held []heldFirstBuild `json:"held,omitempty"` -} - -// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none. -func announcedOf(m inventory.BatchedMerge) keptMerge { - var k keptMerge - _ = json.Unmarshal(m.Event, &k) - return k -} - -// spelledOf is a kept merge's repository as the forge spelled it; empty when its announcement does not say. -func spelledOf(m inventory.BatchedMerge) string { - if e := announcedOf(m); e.Owner != "" { - return e.Owner + "/" + e.Repo - } - return "" -} - -// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced. -func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge { - k := announcedOf(m) - return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves, - Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard} -} - -// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard. -func laterMerge(a, b inventory.BatchedMerge) bool { - if !a.Merged.Equal(b.Merged) { - return a.Merged.After(b.Merged) - } - return a.Heard.After(b.Heard) -} - -// newestCommit is the commit merged last among those a walk carries. -func newestCommit(commits []inventory.PlanCommit) inventory.PlanCommit { - newest := commits[0] - for _, c := range commits[1:] { - if c.Merged.After(newest.Merged) { - newest = c - } - } - return newest -} - -// nameMerges writes the merges a walk answers onto its record, after a merge heard late was given to it. -func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error { - merges, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return err - } - // The walk's own commits stand: a merge heard late is carried by the one of its repository's branch. - var named []inventory.PlanMerge - for _, m := range merges { - repo := m.Repository - if e := spelledOf(m); e != "" { - repo = e - } - n := namedOf(m, repo) - if c := p.CommitOn(m.Repository, m.Branch); c != "" && c != m.Commit { - n.Carried = c - } - named = append(named, n) - } - sort.SliceStable(named, func(i, j int) bool { - return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository) - }) - if p.Delivery == nil { - p.Delivery = &inventory.PlanDelivery{} - } - p.Delivery.Merges = named - return inv.SavePlan(ctx, p) -} - -// cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search -// for the merge that brought the failure; with no walk open, the next merge of that search walked alone; else -// each closed batch cut into a walk with the waiting walks it is related to folded in, unless it is related to a -// walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits. -func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error { - inv := open.inventory - // A merge found in two records is the half-done move of a crash: put right before anything is decided from it. - if repaired, err := inv.RepairMerges(ctx); err != nil { - return err - } else if repaired > 0 { - fmt.Printf("%d merge(s) found in two plans, the half of a move a restart interrupted, are kept by the plan they were moved to\n", repaired) - } - if err := searchAfterFailures(ctx, inv); err != nil { - return err - } - alone, err := inv.AloneMerges(ctx) - if err != nil { - return err - } - if alone, err = coverAlone(ctx, inv, alone); err != nil { - return err - } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return err - } - var started, waiting []inventory.Plan - for i := range plans { - p := plans[i] - if p.Release != nil { - continue // the backlog's walk through the gates: no merge's - } - if p.Waiting() { - waiting = append(waiting, p) - } else { - started = append(started, p) - } - } - batches, err := inv.Batches(ctx) - if err != nil { - return err - } - // **A hold is looked at again at every cut** (ADR 0418 clause 3): a group whose last member merged or closed - // since is released, and its window runs again from now. - if batches, err = releaseHolds(ctx, inv, batches, now); err != nil { - return err - } - keepAll := func(behind string) error { - for i := range batches { - if err := keepBatch(ctx, inv, &batches[i], now, behind); err != nil { - return err - } - } - return nil - } - if len(alone) > 0 { - // A failed walk's search walks its merges alone, one at a time: everything waits for it (kept as it was, - // issue 973). - if len(started) > 0 { - return keepAll(started[0].ID) - } - if len(waiting) > 0 { - // The waiting walk is the one open; the search walks after it, and the batches wait behind both. - return keepAll(waiting[0].ID) - } - return walkAlone(ctx, open, alone[0], now) - } - return cutUnrelated(ctx, open, batches, started, waiting, now) -} - -// releaseHolds asks mesh-delivery again about each group a batch holds, and clears the holds of those with no member -// left to merge. A group mesh-delivery cannot be asked about stays held: the safe error is to wait. It answers the -// batches as kept. -func releaseHolds(ctx context.Context, inv *inventory.Inventory, batches []inventory.Plan, now time.Time) ([]inventory.Plan, error) { - var entries []inventory.Entry - for i := range batches { - b := &batches[i] - if b.Delivery == nil || b.Delivery.Batch == nil || len(b.Delivery.Batch.Holds) == 0 { - continue - } - if entries == nil { - var err error - if entries, err = inv.Catalogued(ctx); err != nil { - return nil, err - } - } - changed := false - for head := range b.Delivery.Batch.Holds { - open, isGroup, _, err := openMembersOf(ctx, inv, deliverySeatHeld(entries), head, "") - if err != nil || (isGroup && len(open) > 0) { - continue - } - holdFor(b, head, "", false, false, now) - changed = true - fmt.Printf("%s: group %s has no member left to merge; its window runs again from now\n", b.ID, head) - } - if changed { - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - return nil, err - } - } - } - return batches, nil -} - -// cutUnrelated cuts the closed batches that share nothing with an open walk, each into a walk beside the open -// ones, and says the others queued behind the first walk they are related to (novox/hq ADR 0407 decision 4, -// issue 973). A batch cut in this look counts as open for the batches after it. -// -// - An own-path batch goes first and folds no waiting walk; a catalogue batch folds the waiting walks it is -// related to, and those it shares nothing with keep waiting. -// - **The controller's own path is related to everything** (decided for issue 973): a walk or batch that -// replaces the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside -// another, and nothing runs beside it. -// - A batch whose window is still open is assembling; one related to an open walk is queued behind it, and -// ADR 0276 decision 4 stands for it: nothing supersedes a walk that started. -func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting []inventory.Plan, now time.Time) error { - inv := open.inventory - sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() }) - var entries []inventory.Entry - var read map[string][]inventory.ReadRepository - var edges []inventory.Edge - judged := false - onPath, err := ownPathNow(ctx, inv) - if err != nil { - return err - } - openWalks, err := openWalksOf(ctx, inv, started, onPath) - if err != nil { - return err - } - for i := range batches { - b := &batches[i] - if !windowClosed(*b, now) { - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - return err - } - continue - } - if !judged { - var err error - if entries, err = inv.Catalogued(ctx); err != nil { - return err - } - if read, err = readForPlanning(ctx, inv); err != nil { - return err - } - if edges, err = inv.Dependencies(ctx); err != nil { - return err - } - judged = true - } - held, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return err - } - fp := footprintAsCut(held, entries, read, edges) - var fold, keep []inventory.Plan - if !b.OwnPath() { - for _, w := range waiting { - ww, err := openWalkOf(ctx, inv, w, onPath) - if err != nil { - return err - } - if ok, _ := related(fp, ww.fp, edges); ok { - fold = append(fold, w) - fp = unionFootprint(fp, ww.fp) - } else { - keep = append(keep, w) - } - } - } - if behind, why := relatedOpen(fp, b.OwnPath(), openWalks, edges); behind != "" { - if err := keepBatchBecause(ctx, inv, b, now, behind, why); err != nil { - return err - } - continue - } - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - return err - } - if err := cutBatch(ctx, open, b, fold, now); err != nil { - return err - } - waiting = keep - if c := windowClockOf(); c != nil { - c.Forget(ctx, b.ID) // a batch cut has no window left to close - } - cut, err := inv.PlanByID(ctx, b.ID) - if err != nil { - return err - } - if !cut.Open() { - continue - } - w, err := openWalkOf(ctx, inv, cut, onPath) - if err != nil { - return err - } - if cut.Waiting() { - waiting = append(waiting, cut) - } else { - openWalks = append(openWalks, w) - } - } - return nil -} - -// cutBatch makes a closed batch one walk, folding in the walks that wait for their word: it keeps its id, and -// the folded walks name it as the walk that took them over. -func cutBatch(ctx context.Context, open *stores, batch *inventory.Plan, waiting []inventory.Plan, now time.Time) error { - inv := open.inventory - var carry []string - for _, w := range waiting { - merges, err := inv.MergesOf(ctx, w.ID) - if err != nil { - return err - } - for _, m := range merges { - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, batch.ID, m.Alone); err != nil { - return err - } - } - // What the folded walk was to build is built by the batch's: its merges were acted on when it was cut, - // so they read as history now. - for name := range w.Modules { - carry = append(carry, name) - } - } - if err := planBatch(ctx, open, batch, carry, now); err != nil { - return err - } - // Closed after the batch's walk is kept, never before: a controller replaced between the two leaves both - // open, which the next cut settles, rather than neither. - for i := range waiting { - w := waiting[i] - w.State = inventory.PlanSuperseded - if w.Delivery == nil { - w.Delivery = &inventory.PlanDelivery{} - } - w.Delivery.TakenOverBy = batch.ID - w.Note = fmt.Sprintf("superseded at tier %d by %s (%s) before it started: that walk answers its merges", - w.Tier, batch.ID, batch.Named()) - if err := inv.SavePlan(ctx, &w); err != nil { - return err - } - fmt.Printf(" %s is %s\n", w.ID, w.Note) - } - return nil -} - -// walkAlone walks one merge on its own commit (ADR 0276 decision 3): a plan of its own, cut at once. -func walkAlone(ctx context.Context, open *stores, m inventory.BatchedMerge, now time.Time) error { - inv := open.inventory - p := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: m.Repository, Branch: m.Branch, - Commit: m.Commit, Merged: m.Merged, Created: now, State: inventory.PlanQueued, Tiers: [][]string{}, - Modules: map[string]*inventory.PlanModule{}, Note: "walked alone after a failed walk carried it", - Delivery: &inventory.PlanDelivery{Alone: true}} - if err := inv.SavePlan(ctx, &p); err != nil { - return err - } - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil { - return err - } - fmt.Printf("%s %.8s, carried by a walk that failed, is walked on its own commit as %s\n", m.Repository, m.Commit, p.ID) - return planBatch(ctx, open, &p, nil, now) -} - -// planBatch plans a batch's merges as one walk, under the batch's id: per repository the latest merge of its -// branch with every file the batch's merges of it changed, the moved modules and their dependents in tiers -// along the graph and the order of any delivery group among them. -func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry []string, now time.Time) error { - inv := open.inventory - merges, err := inv.MergesOf(ctx, batch.ID) - if err != nil { - return err - } - if len(merges) == 0 { - batch.State = inventory.PlanSuperseded - batch.Note = "a batch that holds no merge: nothing to walk" - batch.Delivery.Batch = nil - return inv.SavePlan(ctx, batch) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return err - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return err - } - commits, named := carriedOf(merges) - var moved []string - var members []batchMember - unchanged := map[string]string{} - for _, m := range combinedMerges(merges) { - // **News, whatever was seen since** (review of this change): each merge was judged when it was heard, and - // a cut, a fold or a failed walk since marks its modules seen — so a merge walked alone after a failed - // walk, or folded, would read as history and move nothing. When it was made is not asked again here. - m.MergedAt = "" - // **Unchanged, said** (novox/hq issue 611): read before the merge is acted on, from the same records. - for n, why := range unchangedOfMerge(m, entries, read) { - unchanged[n] = why - } - names, err := movesOfMerge(ctx, inv, m, entries, read) - if err != nil { - return err - } - members = append(members, batchMember{merge: m, moved: names}) - for _, n := range names { - if !slices.Contains(moved, n) { - moved = append(moved, n) - } - } - } - // **Whether it waits for its delivery's word** is read from what its merges moved (novox/hq ADR 0239), never - // from what a folded walk carried along, which holds dependents too. - awaits := awaitsFor(entries, moved) - held := map[string]bool{} - for _, e := range entries { - held[e.Manifest.Module] = true - } - var also []string - for _, n := range carry { - // One the catalogue no longer holds would fail the walk's ask; it is nobody's to build. - if held[n] && !slices.Contains(moved, n) { - moved = append(moved, n) - also = append(also, n) - } - } - sort.Strings(also) - ordered, together, err := changeOrderEdges(ctx, merges, members, entries, moved, edges) - if err != nil { - return err - } - var withheld []string - plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...)) - placedBy(&plan, ordered) - placedTogether(&plan, together) - // **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the - // merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it - // would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change, - // and the stop named it. - if batch.Delivery != nil && batch.Delivery.Batch != nil { - if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 { - withheld = out - fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", ")) - } - } - plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch - plan.Commits = commits - newest := newestCommit(commits) - plan.Repository, plan.Branch, plan.Commit, plan.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged - plan.Delivery = awaits - if plan.Delivery == nil { - plan.Delivery = &inventory.PlanDelivery{} - } - plan.Delivery.Merges = named - plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone - plan.Delivery.Withheld = withheld - plan.Delivery.Order = ordered - // **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as - // the build the mesh holds. - for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) { - unchanged[n] = why - } - plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules) - // **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on. - plan.Times = walkTimesAtCut(*batch, plan, entries, now) - if len(moved) == 0 || len(plan.Modules) == 0 { - plan.State = inventory.PlanDone - plan.Tiers = [][]string{} - plan.Note = "moved nothing the mesh holds: nothing to walk" - if len(plan.Delivery.Unchanged) > 0 { - plan.Note = "moved nothing the mesh holds: nothing to walk; " + unchangedSaid(plan.Delivery.Unchanged) - } - if err := inv.SavePlan(ctx, &plan); err != nil { - return err - } - *batch = plan - fmt.Printf("%s (%s): its merges move nothing the mesh holds\n", plan.ID, plan.Named()) - return nil - } - if hasCycle(plan.Tiers, edges) { - fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n", - strings.Join(plan.Tiers[len(plan.Tiers)-1], ", ")) - } - if len(also) > 0 { - fmt.Printf(" %s, left unbuilt by a walk folded into it, are planned here again\n", strings.Join(also, ", ")) - } - var tiers []string - for i, t := range plan.Tiers { - var names []string - for _, n := range t { - if m := plan.Modules[n]; m != nil && m.Placed != "" { - n += " " + m.Placed - } - names = append(names, n) - } - tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(names, ", "))) - } - var answers []string - for _, n := range named { - answers = append(answers, n.Repository+"@"+short(n.Commit)) - } - fmt.Printf("%s cut: %s, answering %s; %d module(s) in %d tier(s)\n %s\n", plan.ID, plan.Named(), - strings.Join(answers, ", "), len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n ")) - if plan.Waiting() { - plan.Note = waitingNote(plan) - if err := inv.SavePlan(ctx, &plan); err != nil { - return err - } - *batch = plan - fmt.Printf(" %s waits for %s's word before its first tier is asked\n", plan.ID, plan.Delivery.Awaits) - return nil - } - // Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold. - if err := askFirstTierAtCut(&plan, - func(p *inventory.Plan) (bool, error) { return backupsFirst(ctx, inv, p) }, - func() error { return askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]) }); err != nil { - return err - } - if err := inv.SavePlan(ctx, &plan); err != nil { - return err - } - *batch = plan - return nil -} - -// askFirstTierAtCut asks a freshly cut plan's first tier, after the backups of the data it moves (novox/hq ADR 0404, -// issue 1086): the cut asks the tier itself, so advanceOnce, which asks the backups before a first tier, finds it -// asked and never takes them. A plan whose backups wait, failed or could not be worked out asks nothing; it is saved -// unasked and the walk's own step goes on from its record. -func askFirstTierAtCut(p *inventory.Plan, backups func(*inventory.Plan) (bool, error), ask func() error) error { - proceed, err := backups(p) - if err != nil { - p.Note = backupsNote + "not asked, the backups of the plan could not be worked out: " + err.Error() + - "; nothing is asked or sent, and the plan tries again" - return nil - } - if !proceed { - return nil - } - return ask() -} - -// placedBy writes on each module of a plan the edges of its order that put it after another module of the plan, with the -// rule that drew each (novox/hq ADR 0419 clause 5). -func placedBy(plan *inventory.Plan, ordered []inventory.Edge) { - for _, e := range ordered { - m, in := plan.Modules[e.From] - if _, before := plan.Modules[e.To]; !in || !before { - continue - } - said := "after " + e.To + " (" + e.Why + ")" - if m.Placed == "" { - m.Placed = said - } else if !strings.Contains(m.Placed, said) { - m.Placed += "; " + said - } - } -} - -// placedTogether writes the switch's line on the controller's and the node-engine's tier (ADR 0539 §11), so the plan -// says why they share one: the record and each edge overruled. -func placedTogether(plan *inventory.Plan, together string) { - if together == "" { - return - } - for _, n := range []string{theController, theEngine} { - if m, ok := plan.Modules[n]; ok { - m.Placed = strings.Trim(m.Placed+"; "+together, "; ") - } - } -} - -// unchangedSaid names the modules a walk says matched what runs, for its note. -func unchangedSaid(unchanged map[string]string) string { - names := make([]string, 0, len(unchanged)) - for n := range unchanged { - names = append(names, n) - } - sort.Strings(names) - return "unchanged, the build the mesh holds is theirs: " + strings.Join(names, ", ") -} - -// unchangedLeft is what a walk says matched what runs: every module a merge left unchanged that the walk does not -// build anyway, as another merge's module or a dependent (novox/hq issue 611). Nil when none is. -func unchangedLeft(unchanged map[string]string, builds map[string]*inventory.PlanModule) map[string]string { - out := map[string]string{} - for n, why := range unchanged { - if _, built := builds[n]; !built { - out[n] = why - } - } - if len(out) == 0 { - return nil - } - return out -} - -// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the -// modules taken. -func withhold(plan *inventory.Plan, names, moved []string) []string { - var out []string - for _, n := range names { - if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) { - delete(plan.Modules, n) - out = append(out, n) - } - } - if len(out) == 0 { - return nil - } - var tiers [][]string - for _, t := range plan.Tiers { - var kept []string - for _, n := range t { - if !slices.Contains(out, n) { - kept = append(kept, n) - } - } - if len(kept) > 0 { - tiers = append(tiers, kept) - } - } - if tiers == nil { - tiers = [][]string{} - } - plan.Tiers = tiers - sort.Strings(out) - return out -} - -// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window -// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its -// class. A merge walked alone had no window. -func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now time.Time) *inventory.PlanTimes { - cut := now - t := &inventory.PlanTimes{Cut: &cut, Class: classOf(walk, entries)} - if batch.Delivery != nil && batch.Delivery.Batch != nil { - w := batch.Delivery.Batch - closed := w.ClosesAt - if !w.AtMost.IsZero() && w.AtMost.Before(closed) { - closed = w.AtMost - } - if w.CutAtOnce && !batch.Created.IsZero() { - closed = batch.Created // closed when it was made (ADR 0304 decision 1) - } - if !closed.IsZero() { - if closed.After(now) { - closed = now - } - t.WindowClosed = &closed - } - } - return t -} - -// resolverSeat is the seat of the mesh's resolver: a module claiming it is a core module (ADR 0282 decision 1). -const resolverSeat = "mesh-dns-resolver" - -// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own -// path or one holding the mesh's resolver, leaf otherwise. -func classOf(walk inventory.Plan, entries []inventory.Entry) string { - modules := make([]string, 0, len(walk.Modules)) - for m := range walk.Modules { - modules = append(modules, m) - } - return classOfModules(modules, entries) -} - -// classOfModules is the class of modules moved together: core when one is on the controller's own path or holds -// the mesh's resolver, leaf otherwise (ADR 0282 decision 1). -func classOfModules(modules []string, entries []inventory.Entry) string { - resolvers := map[string]bool{} - for _, e := range entries { - for _, c := range e.Manifest.Claims { - if c.Name == resolverSeat { - resolvers[e.Manifest.Module] = true - } - } - } - onPath := ownPathOf(entries) - for _, m := range modules { - if _, own := onPath[m]; own || resolvers[m] { - return inventory.ClassCore - } - } - return inventory.ClassLeaf -} - -// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the -// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is -// removed when the last merge of the batch that changed it removed it. merges are in the order they were made. -func combinedMerges(merges []inventory.BatchedMerge) []link.SourceMoved { - type combined struct { - latest inventory.BatchedMerge - m link.SourceMoved - said bool - removed map[string]bool - paths []string - dirs []string - cut bool - } - by := map[string]*combined{} - var keys []string - for _, b := range merges { - var e link.SourceMoved - if err := json.Unmarshal(b.Event, &e); err != nil { - continue - } - k := strings.ToLower(b.Repository) + "\x00" + b.Branch - c, ok := by[k] - if !ok { - c = &combined{latest: b, m: e, said: true, removed: map[string]bool{}} - by[k] = c - keys = append(keys, k) - } else if laterMerge(b, c.latest) { - c.latest, c.m = b, e - } - c.said = c.said && e.ModuleDirsSaid - c.cut = c.cut || e.PathsTruncated - c.paths, c.dirs = union(c.paths, e.Paths), union(c.dirs, e.ModuleDirs) - for _, path := range e.Paths { - c.removed[path] = slices.Contains(e.Removed, path) - } - for _, path := range e.Removed { - c.removed[path] = true - } - } - sort.Strings(keys) - var out []link.SourceMoved - for _, k := range keys { - c := by[k] - m := c.m - m.Paths, m.ModuleDirs, m.PathsTruncated, m.ModuleDirsSaid = c.paths, c.dirs, c.cut, c.said - m.Removed = nil - for _, path := range c.paths { - if c.removed[path] { - m.Removed = append(m.Removed, path) - } - } - out = append(out, m) - } - return out -} - -// union is a and b without repeats, in order. -func union(a, b []string) []string { - out := append([]string{}, a...) - for _, s := range b { - if !slices.Contains(out, s) { - out = append(out, s) - } - } - return out -} - -// batchMember is one repository's merge in a batch and what it moves. -type batchMember struct { - merge link.SourceMoved - moved []string -} - -// changeOrderEdges is the order of a batch's moved modules as edges of its plan's tiers (novox/hq ADR 0419): drawn -// per module from the module graph and from what the merges say of the manifests they touch (delivery_edges.go), -// whatever repository the merges are in and whether or not they form a delivery group. No `after:` line is read. -// A cycle is said, never lost: the plan puts the modules left in its last tier, and the composed check fails the -// group before it merges. -func changeOrderEdges(ctx context.Context, merges []inventory.BatchedMerge, members []batchMember, - entries []inventory.Entry, moved []string, edges []inventory.Edge) ([]inventory.Edge, string, error) { - // What each repository's branch held before the batch: the first parent of its earliest merge in it. - earliest := map[string]inventory.BatchedMerge{} - for _, k := range merges { - key := strings.ToLower(k.Repository) + "\x00" + k.Branch - if e, ok := earliest[key]; !ok || laterMerge(e, k) { - earliest[key] = k - } - } - var sources []changeSource - for _, m := range members { - s := changeSource{Owner: m.merge.Owner, Repo: m.merge.Repo, New: m.merge.Commit, - Paths: m.merge.Paths, Truncated: m.merge.PathsTruncated, Removed: m.merge.Removed} - if e, ok := earliest[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)+"\x00"+m.merge.Base]; ok { - s.Since = e.Commit - } - if s.Truncated { - s.Paths, s.Truncated = truncatedPaths(entries, s.Owner, s.Repo, moved), false - } - sources = append(sources, s) - } - changes, err := changesOf(ctx, sources) - if err != nil { - return nil, "", fmt.Errorf("what the merges say of their manifests cannot be read, so the order of their modules "+ - "cannot be drawn: %w", err) - } - named := false - for _, k := range merges { - var e link.SourceMoved - if json.Unmarshal(k.Event, &e) == nil && switchNamed(e.Body) { - named = true - } - } - order := orderEdges(moved, changes, edges, named) - for _, e := range order.Edges { - fmt.Printf(" order: %s\n", edgeSaid(e)) - } - if order.Together != "" { - fmt.Printf(" order: %s\n", order.Together) - } - if said := order.CycleSaid(); said != "" { - fmt.Printf(" %s\n", said) - } - return order.Edges, order.Together, nil -} - -// searchAfterFailures starts the search a failed walk leaves (ADR 0276 decision 3): every merge it answered -// in a later commit of its repository waits to be walked on its own commit. A stopped walk starts none: a -// person ended it. -func searchAfterFailures(ctx context.Context, inv *inventory.Inventory) error { - plans, err := inv.RecentPlans(ctx, 20) - if err != nil { - return err - } - for _, p := range plans { - if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped != "" || - strings.HasPrefix(p.Note, "stopped by hand") || strings.HasPrefix(p.Note, "closed by hand") { - continue - } - merges, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return err - } - for _, m := range merges { - if m.Alone || p.CommitOf(m.Repository) == m.Commit { - continue // its own commit was walked - } - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, "", true); err != nil { - return err - } - fmt.Printf("%s failed; %s %.8s, which it carried in %.8s, is walked on its own commit\n", p.ID, - m.Repository, m.Commit, p.CommitOf(m.Repository)) - } - } - return nil -} - -// coverAlone gives every merge waiting to be walked alone to a walk done that holds a later merge of its -// branch — one walked alone, or the failed walk retried by a person: the search stops at the first merge -// delivered, and that walk answers the older ones. What is left waits, the newest first. -func coverAlone(ctx context.Context, inv *inventory.Inventory, alone []inventory.BatchedMerge) ([]inventory.BatchedMerge, error) { - var left []inventory.BatchedMerge - for _, m := range alone { - later, err := inv.LaterMergesOf(ctx, m.Repository, m.Branch, m.Merged) - if err != nil { - return nil, err - } - covered := false - for _, l := range later { - p, err := inv.PlanByID(ctx, l.Plan) - if err != nil { - return nil, err - } - if p.State != inventory.PlanDone { - continue - } - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil { - return nil, err - } - if err := nameMerges(ctx, inv, &p); err != nil { - return nil, err - } - fmt.Printf("%s %.8s is answered by %s, which delivered a later merge of its branch on its own commit\n", - m.Repository, m.Commit, p.ID) - covered = true - break - } - if !covered { - left = append(left, m) - } - } - return left, nil -} - -// batchWords is a batch in the operator's words (ADR 0276 decision 7): "assembling: 13 s left (at the latest -// 00:31:09); grouped: novox/mesh-catalog@48bda475 (answers 553b7191), …; plan not yet calculated". -func batchWords(b inventory.Plan, now time.Time) string { - grouped := groupedWords(b) - if b.Delivery == nil || b.Delivery.Batch == nil { - return "grouped: " + grouped + "; plan not yet calculated" - } - w := b.Delivery.Batch - if w.Own { - grouped += " (on the controller's own path)" - } - if b.State == inventory.PlanQueued { - because := "" - if w.BehindWhy != "" { - because = " (" + w.BehindWhy + ")" - } - return fmt.Sprintf("queued behind %s%s; grouped: %s; plan not yet calculated", w.Behind, because, grouped) - } - if w.CutAtOnce { - return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated", - grouped) - } - left := w.ClosesAt.Sub(now) - if w.AtMost.Before(w.ClosesAt) { - left = w.AtMost.Sub(now) - } - if left < 0 { - left = 0 - } - waits := "" - if w.Waits != "" { - waits = " — " + w.Waits - } - return fmt.Sprintf("assembling: %s left (at the latest %s)%s; grouped: %s; plan not yet calculated", - secondsWords(left), w.AtMost.Local().Format("15:04:05"), waits, grouped) -} - -// groupedWords is a batch's commits, each with the earlier merges it answers. -func groupedWords(b inventory.Plan) string { - var out []string - for _, c := range b.Carried() { - s := c.Repository + "@" + short(c.Commit) - var answers []string - if b.Delivery != nil { - for _, m := range b.Delivery.Merges { - if strings.EqualFold(m.Repository, c.Repository) && m.Carried == c.Commit { - answers = append(answers, short(m.Commit)) - } - } - } - if len(answers) > 0 { - s += " (answers " + strings.Join(answers, ", ") + ")" - } - out = append(out, s) - } - if len(out) == 0 { - return "nothing yet" - } - return strings.Join(out, ", ") -} - -// batchLines is a batch's grouped list as `plans` prints it under its line (asked by the operator, 2026-10-10): -// one line per repository, its pull request and title, the earlier merges it answers, and what its merges move. -func batchLines(b inventory.Plan) []string { - var out []string - for _, c := range b.Carried() { - var head *inventory.PlanMerge - var answers []string - moves := map[string]bool{} - if b.Delivery != nil { - for i := range b.Delivery.Merges { - m := &b.Delivery.Merges[i] - if !strings.EqualFold(m.Repository, c.Repository) { - continue - } - for _, n := range m.Moves { - moves[n] = true - } - switch { - case m.Commit == c.Commit: - head = m - case m.Carried == c.Commit: - answers = append(answers, pullWords(*m)) - } - } - } - line := repoName(c.Repository) + " " + short(c.Commit) - if head != nil && head.Number > 0 { - line = repoName(c.Repository) + " " + pullWords(*head) - } - if len(answers) > 0 { - line += " (answers " + strings.Join(answers, ", ") + ")" - } - if len(moves) > 0 { - line += " · " + strings.Join(sortedKeysOf(boolsToStrings(moves)), ", ") - } - out = append(out, line) - } - return out -} - -// pullWords is a merge as its pull request: "#175 a tap shows its outcome", the title cut at fifty runes; the -// commit where the announcement named no pull request. -func pullWords(m inventory.PlanMerge) string { - if m.Number == 0 { - return short(m.Commit) - } - s := fmt.Sprintf("#%d", m.Number) - if t := cutTitle(m.Title, 50); t != "" { - s += " " + t - } - return s -} - -// cutTitle is a title cut at n runes, with an ellipsis where it was cut. -func cutTitle(title string, n int) string { - r := []rune(strings.TrimSpace(title)) - if len(r) <= n { - return string(r) - } - return strings.TrimSpace(string(r[:n-1])) + "…" -} - -// boolsToStrings is a set's members, for sortedKeysOf. -func boolsToStrings(set map[string]bool) map[string]string { - out := make(map[string]string, len(set)) - for k := range set { - out[k] = "" - } - return out -} - -// secondsWords is a short wait as a person reads it. -func secondsWords(d time.Duration) string { - if d < 2*time.Minute { - return fmt.Sprintf("%d s", int(d.Round(time.Second)/time.Second)) - } - return d.Round(time.Second).String() -} - -// walkOrBatch is what a plan record is, in a word. -func walkOrBatch(p inventory.Plan) string { - if p.Batch() { - return "batch" - } - return "walk" -} - -// orWords is s, or the words for when it is empty. -func orWords(s, none string) string { - if s == "" { - return none - } - return s -} diff --git a/cmd/mesh-controller/batches_integration_test.go b/cmd/mesh-controller/batches_integration_test.go deleted file mode 100644 index 38c83a2b..00000000 --- a/cmd/mesh-controller/batches_integration_test.go +++ /dev/null @@ -1,727 +0,0 @@ -//go:build integration - -package main - -import ( - "bytes" - "io" - "os" - "slices" - "strings" - "sync/atomic" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func windowed(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - noGroups(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, m := range []struct{ module, repository, path string }{ - {"app", "novox/mesh-catalog", "modules/app"}, - {"notes", "novox/mesh-catalog", "modules/notes"}, - {"one", "novox/one", ""}, - {"two", "novox/two", ""}, - {"three", "novox/three", ""}, - } { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"}, - inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - } - return open -} - -func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) { - t.Helper() - if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil { - t.Fatal(err) - } -} - -func cutAt(t *testing.T, open *stores, now time.Time) { - t.Helper() - if err := cutBatchesHeld(t.Context(), open, now); err != nil { - t.Fatal(err) - } -} - -func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - claude := catalogueMerge("553b7191claude", "app", t0) - dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, claude, t0.Add(time.Second)) - hear(t, open, dunst, t0.Add(18*time.Second)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { - t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - cutAt(t, open, t0.Add(18*time.Second+89*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window closed: %+v", ws) - } - cutAt(t, open, t0.Add(18*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit { - t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits) - } - for _, m := range []string{"app", "notes"} { - if _, in := w.Modules[m]; !in { - t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules) - } - } - want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit}, - {Repository: "novox/mesh-catalog", Commit: dunst.Commit}} - if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) { - t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want) - } - // The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at - // which commit: the one the branch holds, which is the later merge the walk carries. - for _, m := range w.Tiers[w.Tier] { - if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit { - t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s) - } - } - // Heard again, as the bus may hand it over twice: never a second merge, nor a second walk. - hear(t, open, claude, t0.Add(5*time.Minute)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs)) - } -} - -func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/three") - asksRecorded(t) - for i, r := range []string{"one", "two", "three"} { - hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"), - t0.Add(time.Duration(i)*20*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(41*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 { - t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges) - } - for _, r := range []string{"one", "two", "three"} { - if w.CommitOf("novox/"+r) != "c-"+r { - t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r)) - } - if _, in := w.Modules[r]; !in { - t.Fatalf("the walk does not build %s: %v", r, w.Modules) - } - } - // Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build). - for _, r := range w.Tiers[w.Tier] { - if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r { - t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s) - } - } -} - -func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second)) - cutAt(t, open, t0.Add(100*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatal("the window closed 90 seconds after its first merge, not after its last") - } - cutAt(t, open, t0.Add(170*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 { - t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws) - } - - busy := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - var last time.Time - for i := 0; i < 12; i++ { - last = t0.Add(time.Duration(i) * time.Minute) - repo := []string{"one", "two", "three"}[i%3] - hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last) - if ws, _ := walks(t, busy); len(ws) > 0 { - if i != 10 { - t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i) - } - break - } - } - ws, _ := walks(t, busy) - if len(ws) != 1 { - t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws)) - } -} - -func TestTheAssemblingBatchIsShown(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/553b7191claude") - asksWithPaths(t) - var said []inventory.Plan - was := inventory.PlanSaved - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) } - t.Cleanup(func() { inventory.PlanSaved = was }) - now := time.Now().UTC() - claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second)) - claude.Number, claude.Title = 174, "claude-code: an agent proposes a section" - dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second)) - dunst.Number, dunst.Title = 175, "dunst: the font the operator chose" - hear(t, open, claude, now.Add(-19*time.Second)) - hear(t, open, dunst, now.Add(-time.Second)) - hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now) - out := captured(t, func() error { return plansCommand(t.Context(), nil) }) - lines := strings.Split(out, "\n") - first := lines[0] - for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " + - "(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} { - if !strings.Contains(first, want) { - t.Fatalf("plans' first line %q does not say %q", first, want) - } - } - // Under it, one line per repository: the pull request, what it answers, what it moves. - if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+ - "#174 claude-code: an agent proposes a section) · app, notes" || - strings.TrimSpace(lines[2]) != "one a6bc0931 · one" { - t.Fatalf("the grouped list reads %q", lines[1:4]) - } - if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil || - len(said[len(said)-1].Delivery.Merges) != 3 { - t.Fatalf("the batch was not said as assembling with its merges: %+v", said) - } -} - -func captured(t *testing.T, run func() error) string { - t.Helper() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - was := os.Stdout - os.Stdout = w - runErr := run() - os.Stdout = was - _ = w.Close() - var b bytes.Buffer - _, _ = io.Copy(&b, r) - if runErr != nil { - t.Fatal(runErr) - } - return b.String() -} - -func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("the first batch was not cut: %+v", ws) - } - first := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(5*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID { - t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs) - } - if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() { - t.Fatalf("the started walk was %s by the next merge", got.State) - } - if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) { - t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute))) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(t.Context(), &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(6*time.Minute)) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { - t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs) - } -} - -func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Waiting() { - t.Fatalf("the walk does not wait for its word: %+v", ws) - } - waiting := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(2*time.Minute+90*time.Second)) - ws, bs := walks(t, open) - if len(bs) != 0 || len(ws) != 2 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - folded, err := open.inventory.PlanByID(ctx, waiting.ID) - if err != nil { - t.Fatal(err) - } - newer := ws[0] - if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID { - t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy) - } - if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" { - t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges) - } - if _, in := newer.Modules["one"]; !in { - t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules) - } -} - -func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, later, t0.Add(18*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - // Moves only notes, which the done walk built from the branch after it: answered there, at once. - hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 || - !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early", - Carried: later.Commit}) { - t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges) - } - // Moves app, which that walk never built: the next batch walks it. - hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" { - t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs) - } -} - -func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - // Made before the cut marks their modules seen, as merges are: walked again, they are news all the same. - oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute)) - middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second)) - newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second)) - for i, m := range []link.SourceMoved{oldest, middle, newest} { - hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - failed := ws[0] - failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0" - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, failed.ID) - if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 { - t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges) - } - ws, _ = walks(t, open) - alone := ws[0] - if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 || - !sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) { - t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone) - } - // The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the - // module is taken from its key at the commit the merge was walked on. - if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit { - t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules) - } - // Retried, the failed walk would name merges the search answers now. - if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") { - t.Fatalf("a failed walk whose merges are searched was retried: %v", err) - } - alone.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &alone); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(4*time.Minute)) - ws, bs := walks(t, open) - if ws[0].ID != alone.ID || len(bs) != 0 { - t.Fatalf("the search went on after a merge was delivered: %+v", ws[0]) - } - done, _ := open.inventory.PlanByID(ctx, alone.ID) - if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", - Commit: oldest.Commit, Carried: middle.Commit}) { - t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges) - } - // A stopped walk starts no search: a person ended it. - hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute)) - hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second)) - cutAt(t, open, t0.Add(8*time.Minute)) - ws, _ = walks(t, open) - if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(9*time.Minute)) - if again, _ := walks(t, open); again[0].ID != ws[0].ID { - t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0]) - } -} - -func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) { - for _, grouped := range []bool{true, false} { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []string{"mesh-host"} { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - host := repoMerge("mesh-host", "h1", t0) - ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second)) - host.Head, ctl.Head = "feat/together", "feat/together" - if !grouped { - ctl.Head = "feat/alone" - } - hear(t, open, ctl, t0.Add(2*time.Second)) - hear(t, open, host, t0.Add(3*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walks", len(ws)) - } - tiers := ws[0].Tiers - if grouped { - if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) || - !slices.Equal(tiers[1], []string{"mesh-controller"}) { - t.Fatalf("the group's order is not the walk's tiers: %v", tiers) - } - } else if len(tiers) != 1 { - t.Fatalf("two merges of no group were ordered: %v", tiers) - } - } -} - -func TestABatchSurvivesARestart(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second)) - - // A new controller: nothing of the first one's but the store. - again, err := openStores(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(again.Close) - hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again - cutAt(t, again, t0.Add(119*time.Second)) - if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs) - } - cutAt(t, again, t0.Add(120*time.Second)) - ws, bs := walks(t, again) - if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs) - } -} - -func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - c2 := repoMerge("one", "c2", t0.Add(-50*time.Second)) - hear(t, open, c2, t0) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one", - Commit: "c1", Carried: "c2"}) { - t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges) - } - - // A walk of two that built only two: a late merge of one is not its to name. - hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute)) - cutAt(t, open, t0.Add(18*time.Minute)) - ws, _ = walks(t, open) - w = ws[0] - delete(w.Modules, "two") - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute)) - if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 { - t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges) - } - if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" { - t.Fatalf("the late merge did not join the next batch: %+v", bs) - } -} - -func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - // The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with - // the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other. - wholeKeys, held := openPlanKeys, &atomic.Bool{} - held.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err - } - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(2*time.Minute)) - started, _ := walks(t, open) - waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0, - State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatalf("the word was refused beside %s: %v", started[0].ID, err) - } - before := len(*asked) - advanceHeld(ctx, open) - got, _ := open.inventory.PlanByID(ctx, waiting.ID) - if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) { - t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note) - } - // Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3. - later := time.Now().Add(time.Hour) - if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") { - t.Fatalf("a deferred walk reads %q", line) - } - facts, _, err := gatherPlans(ctx, open.inventory, later, nil) - if err != nil { - t.Fatal(err) - } - for _, f := range facts { - if f.id == got.ID { - t.Fatalf("a deferred walk is watched as a tier running late: %+v", f) - } - } - // Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale. - done, err := open.inventory.PlanByID(ctx, started[0].ID) - if err != nil { - t.Fatal(err) - } - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) != before+1 { - t.Fatalf("the walk did not start once the started one ended: asked %v", *asked) - } -} - -func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0) - hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second)) - hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second)) - _, bs := walks(t, open) - var ownBatch, catalogueBatch inventory.Plan - for _, b := range bs { - if b.OwnPath() { - ownBatch = b - } else { - catalogueBatch = b - } - } - if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 || - len(ownBatch.Delivery.Merges) != 1 { - t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs) - } - if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") { - t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second))) - } - // Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it, - // is cut when the controller's walk ended, and waits for its word with both merges. - cutAt(t, open, t0.Add(2*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" { - t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws) - } - ownWalk := ws[0] - for _, m := range []string{"app", "notes"} { - if _, in := ownWalk.Modules[m]; in { - t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m) - } - } - if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() { - t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs) - } - ownWalk.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - ws, bs = walks(t, open) - if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs) - } - for _, m := range []string{"app", "notes"} { - if _, in := ws[0].Modules[m]; !in { - t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules) - } - } - for _, a := range *asked { - if a[1] == "modules/app" || a[1] == "modules/notes" { - t.Fatalf("a catalogue module was asked without the word: %v", *asked) - } - } - - // A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path - // walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended. - catalogueWalk := ws[0] - // The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the - // stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk. - wholeKeys, held := openPlanKeys, &atomic.Bool{} - held.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err - } - hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID) - if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() { - t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws) - } - if _, in := ws[0].Modules["app"]; in { - t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules) - } - if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatal(err) - } - before := len(*asked) - advanceHeld(ctx, open) - if len(*asked) != before { - t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:]) - } - // Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale. - own2, err := open.inventory.PlanByID(ctx, ws[0].ID) - if err != nil { - t.Fatal(err) - } - own2.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &own2); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" { - t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:]) - } -} - -func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []struct { - module string - claims []catalogue.Claim - }{ - {"nats", nil}, - {"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - } { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano), - Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"}, - ModuleDirsSaid: true} - hear(t, open, mixed, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() { - t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws) - } - done := ws[0] - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, done.ID) - _, bs := walks(t, open) - if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" { - t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)", - got.Delivery.Merges, bs) - } -} diff --git a/cmd/mesh-controller/batches_test.go b/cmd/mesh-controller/batches_test.go deleted file mode 100644 index cbbdc11b..00000000 --- a/cmd/mesh-controller/batches_test.go +++ /dev/null @@ -1,293 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "os" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a -// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge -// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings. -func init() { - mergeWindowDefault = 0 - // No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change - // (delivery_edges.go). A test that wants some replaces these (forgeHolding). - manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) { - return catalogue.Manifest{}, false, nil - } - parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } -} - -// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its -// settings file says them, and -// modules built from the catalogue (app, notes) and from three repositories of their own. - -// t0 is the moment a test's clock starts: merges are dated by it, and heard after it. -var t0 = time.Now().UTC().Truncate(time.Second) - -// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull -// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group -// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test -// whose merges wait for the window as a group's says so (groupsWaiting). -func catalogueMerge(commit, module string, made time.Time) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit, - MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"}, - ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true} -} - -// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge). -func repoMerge(repo, commit string, made time.Time) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit, - MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}} -} - -// noGroups stubs mesh-delivery's word on groups for a test: no head names a group. -func noGroups(t *testing.T) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named -// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and -// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds -// its batch open however long it takes (ADR 0418 clause 3), is groupStates'. -func groupsWaiting(t *testing.T, heads ...string) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) { - if slices.Contains(heads, head) { - return nil, false, errors.New("the test's mesh-delivery does not say") - } - return nil, false, nil - } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did -// before novox/hq ADR 0304: mesh-delivery cannot say any head's group. -func everyHeadWaits(t *testing.T) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { - return nil, false, errors.New("the test's mesh-delivery does not say") - } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the -// members merge. -func groupStates(t *testing.T, head string, states map[string]string) { - t.Helper() - groupStatesFrom(t, head, states, true) -} - -// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the -// composed check's: only the first holds the group's batch open for its last member. -func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) { - if asked != head { - return nil, false, nil - } - var members []groupMember - for repository, state := range states { - members = append(members, groupMember{Repository: repository, State: state, Composed: composed}) - } - return members, true, nil - } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group, -// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share. -func fromHead(m link.SourceMoved, head string) link.SourceMoved { - m.Head = head - return m -} - -// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is -// related to a batch assembling (ADR 0304 decision 1). -func lone(m link.SourceMoved) link.SourceMoved { - m.Head = "" - return m -} - -// hear is a merge heard at a moment. - -// cutAt is the cutter looking at a moment. - -// walks is every plan record that is a walk, newest first, and the batches not yet cut. -func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) { - t.Helper() - recent, err := open.inventory.RecentPlans(t.Context(), 50) - if err != nil { - t.Fatal(err) - } - for _, p := range recent { - if p.Batch() { - batches = append(batches, p) - } else { - walks = append(walks, p) - } - } - return walks, batches -} - -// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the -// later commit, which names both merges — the earlier carried by the later. - -// Merges of three repositories in one window are one walk, one commit for each. - -// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window -// closes at most ten minutes after its first merge however busy. - -// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest, -// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too. - -// captured is what a command printed. - -// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is -// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4). - -// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it -// names the walk that took it over. - -// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it -// moves — named at once on a walk done — and joins the next batch when it does not. - -// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest -// first; the first delivered answers the older ones, and the search stops. - -// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head -// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier. - -// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the -// store, a merge handed over again is not doubled, and the batch is cut when its window closes. - -// The window's settings reach the controller in its settings file, read at every look; one that says no -// duration, or no file, is the default. -func TestTheWindowIsTheControllersSetting(t *testing.T) { - file := t.TempDir() + "/merge-window.json" - t.Setenv(mergeWindowFileVar, file) - for _, c := range []struct { - content string - window, atMost time.Duration - }{ - {`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute}, - {`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault}, - {`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault}, - } { - if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil { - t.Fatal(err) - } - if w, m := mergeWindowOf(); w != c.window || m != c.atMost { - t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost) - } - } - t.Setenv(mergeWindowFileVar, file+".gone") - if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault { - t.Errorf("no file reads as %s and %s", w, m) - } -} - -// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362). -func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) { - f := calm(t0) - f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery", - since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{ - {Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"}, - {Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}} - got := watchWaits(f) - if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") || - !strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") { - t.Fatalf("the waiting walk does not name both merges: %+v", got) - } -} - -// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since -// the cut, in a repository of one module (review of this change); a walk that never built what it moves names -// nothing, and the merge joins the next batch. - -// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to -// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before. - -// A file a merge of the batch removed and a later one brought back is not removed; one removed last is. -func TestARemovedFileIsTheLastMergesWord(t *testing.T) { - ev := func(commit string, paths, removed []string) inventory.BatchedMerge { - m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed} - b, _ := json.Marshal(m) - return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b, - Merged: t0.Add(time.Duration(len(commit)) * time.Second)} - } - got := combinedMerges([]inventory.BatchedMerge{ - ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}), - ev("bb", []string{"x/module.json"}, nil), - ev("ccc", []string{"z.go"}, nil)}) - if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) || - len(got[0].Paths) != 3 { - t.Fatalf("combined as %+v", got) - } -} - -// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word -// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on -// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk -// still waits for the word, so no catalogue delivery skips its turn. - -// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and -// title, the modules it moves and the machines running them, then the state words; a record naming no pull -// request is named by its commit, and a title is cut at fifty runes. -func TestAPlanLineNamesWhatItMoves(t *testing.T) { - now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1", - State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute), - Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}}, - Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}}, - Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", - Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}} - running := func(module string) []string { - if module == "messenger" || module == "telegram" { - return []string{"novox"} - } - return nil - } - if got, want := planLineOn(p, now, tierAtLeast, running), - "mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want { - t.Fatalf("the line reads %q, want %q", got, want) - } - if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") { - t.Fatalf("without the machines the line reads %q", got) - } - old := p - old.Commits, old.Delivery = nil, nil - if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") { - t.Fatalf("a record naming no pull request reads %q", got) - } - long := p - long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6) - if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") { - t.Fatalf("a long title is not cut at fifty runes: %q", got) - } -} - -// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it. -func sameMerge(a, b inventory.PlanMerge) bool { - return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried -} - -// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched -// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder: -// its delivery would skip its turn. It joins the next catalogue batch. diff --git a/cmd/mesh-controller/batches_watch.go b/cmd/mesh-controller/batches_watch.go deleted file mode 100644 index ae933763..00000000 --- a/cmd/mesh-controller/batches_watch.go +++ /dev/null @@ -1,153 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still -// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18), -// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19). - -// The kinds S18 and S19 raise. -const ( - kindBatchNotCut = "batch-not-cut" - kindBatchBehindWalk = "batch-behind-walk" - // kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after - // its first (novox/hq ADR 0418 clause 3). - kindGroupNotMerged = "group-not-merged" -) - -// batchFacts is one batch not yet cut, as the watchdogs read it. -type batchFacts struct { - id, state, grouped string - // atMost is when its window closes at the latest; closed when it closed. - atMost, closed time.Time - // behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its - // tiers. - behind string - walkBound time.Duration - // holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first - // merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting). - holds map[string]string - created time.Time - atMostWindow time.Duration -} - -// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind. -func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) { - batches, err := inv.Batches(ctx) - if err != nil || len(batches) == 0 { - return nil, err - } - bounds, err := measuredTierBounds(ctx, inv, now) - if err != nil { - return nil, err - } - var out []batchFacts - for _, b := range batches { - _, atMostWindow := mergeWindowOf() - f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow} - if w := b.Delivery; w != nil && w.Batch != nil { - f.holds = w.Batch.Holds - f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind - if f.atMost.Before(f.closed) { - f.closed = f.atMost - } - if w.Batch.CutAtOnce && !b.Created.IsZero() { - // Its window closed when it was made, which is its latest too (ADR 0304 decision 1). - f.atMost, f.closed = b.Created, b.Created - } - } - if f.behind != "" { - if walk, err := inv.PlanByID(ctx, f.behind); err == nil { - f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers))) - } - } - out = append(out, f) - } - return out, nil -} - -// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open. -func watchBatchesNotCut(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, b := range f.batches { - if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter || - len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition - continue - } - late := f.now.Sub(b.atMost) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut, - Severity: conditions.Warning, - Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+ - "the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped), - Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)), - Headline: "Merged changes are not being delivered", - Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+ - "Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)), - Resolved: "Merged changes are being delivered again"}) - } - return out -} - -// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it. -func watchBatchesBehind(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, b := range f.batches { - if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound { - continue - } - in := f.now.Sub(b.closed) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk, - Severity: conditions.Warning, - Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+ - "grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound), - b.grouped, b.behind), - Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)), - Headline: "Merged changes wait behind a slow delivery", - Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+ - "them, which is taking longer than it should. Nothing is lost.", humanDuration(in)), - Resolved: "Merged changes no longer wait behind a slow delivery"}) - } - return out -} - -// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge -// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of -// the group is delivered while it waits. -func watchGroupsNotMerged(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, b := range f.batches { - if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow { - continue - } - heads := make([]string, 0, len(b.holds)) - for h := range b.holds { - heads = append(heads, h) - } - sort.Strings(heads) - var why []string - for _, h := range heads { - why = append(why, b.holds[h]) - } - waited := f.now.Sub(b.created) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, - Kind: kindGroupNotMerged, Severity: conditions.Warning, - Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+ - "of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id), - Said: "a group waits for a member that has not merged", - Headline: "A change waits for the rest of its group to merge", - Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+ - "One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.", - humanDuration(waited)), - Resolved: "The group's last change merged"}) - } - return out -} diff --git a/cmd/mesh-controller/beside_test.go b/cmd/mesh-controller/beside_test.go deleted file mode 100644 index d3626c82..00000000 --- a/cmd/mesh-controller/beside_test.go +++ /dev/null @@ -1,117 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once -// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery. -func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - // Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in - // builder's key done at once, one step would carry a one-module walk to done. - wholeKeys, held := openPlanKeys, &atomic.Bool{} - held.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err - } - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() { - t.Fatalf("the first walk is not open: %+v", ws) - } - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("two", "c2", heard)), heard) - advanceHeld(ctx, open) - ws, bs := walks(t, open) - if len(ws) != 2 || len(bs) != 0 { - t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs)) - } - for _, w := range ws { - if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) { - t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(), - deferred(w), askedAny(w), w.Note) - } - } -} - -// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is -// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973). -func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }() - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("one", "c2", heard)), heard) - hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second)) - ws, bs := walks(t, open) - if len(ws) != 2 || len(bs) != 1 { - t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs)) - } - b := bs[0] - words := batchWords(b, heard) - if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" || - !strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") { - t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words) - } - if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() { - t.Fatalf("the started walk was %s by the related merge", got.State) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, heard.Add(2*time.Second)) - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs) - } -} - -// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever -// the two share (issue 973): it replaces the controller, so it goes alone. -func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - first := ws[0] - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued || - bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") { - t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, heard.Add(time.Second)) - ws, _ = walks(t, open) - if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" { - t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws) - } - own := ws[0] - // Unrelated to the own-path walk by every footprint, a merge still waits for it. - later := heard.Add(time.Minute) - hear(t, open, lone(repoMerge("two", "c2", later)), later) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID { - t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs) - } -} diff --git a/cmd/mesh-controller/bindings.go b/cmd/mesh-controller/bindings.go index aab111f1..64563d50 100644 --- a/cmd/mesh-controller/bindings.go +++ b/cmd/mesh-controller/bindings.go @@ -54,21 +54,6 @@ func otherMachines(machine string, nodes ...string) []string { return out } -// reportKept says every move a machine's resolution refused, on the push composing it, and raises its -// condition at once where this process keeps the conditions: a push is when a person is looking. -func reportKept(ctx context.Context, plan catalogue.Resolution) { - for _, k := range plan.Kept { - fmt.Printf("%s: the mesh %s\n", plan.Node, k) - if conditionsFrom != nil { - o := keptObservation(k) - o.Source = probeBindingsID - if _, err := conditionsFrom.Observe(ctx, o); err != nil { - fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err) - } - } - } -} - // probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last // sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin // moves it (said, so the data goes first), and never anything else. diff --git a/cmd/mesh-controller/bindings_test.go b/cmd/mesh-controller/bindings_test.go index 2101e614..8cd06d61 100644 --- a/cmd/mesh-controller/bindings_test.go +++ b/cmd/mesh-controller/bindings_test.go @@ -75,19 +75,3 @@ func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) { t.Error("two consumers, one condition") } } - -// A push says the move it refused, and raises its condition at once. -func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) { - k, _ := withConditionsInMemory(t) - reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop", - Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"}, - Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}}) - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent || - open[0].Source != probeBindingsID { - t.Fatalf("raised %+v", open) - } -} diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index c6318a61..f4c78b0d 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -150,16 +150,6 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild) return notRegistered(err) } } - // **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard - // twice, or replayed, would otherwise make the build a rollback put back what the module is again, - // and the next push would send it. - if failed, err := inv.GateFailed(ctx, kept.ID); err != nil { - return manifest, err - } else if failed { - return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+ - "back: it is recorded and not registered again — a newer build is", j.by, manifest.Module, - short(j.commit)) - } // Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a // key that list does not hold, or of another type, is refused here and never enters the catalogue. if j.carried != nil { @@ -229,10 +219,6 @@ type answers struct { // pair that answers "has it caught up", which waiting alone cannot (the sent digest is // recorded at send, not at apply). reported []inventory.Reported - // plans is what the last merges produced and where each stands (novox/hq ADR 0162). - plans []inventory.Plan - // tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296). - tierBounds tierBounds // refused is why a machine cannot be worked out at all, by name. A different thing from every // other answer here: those are about a machine that was told something, and this is about one // that cannot be told anything — it never reaches waiting, because nothing was computed for it @@ -287,8 +273,4 @@ type answers struct { // applied, is not well: an assignment somebody made is not made yet, or will not be. pending []inventory.PendingAssignment pendingUnread string - // lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq - // issue 485); lastSentUnread why it could not be read. - lastSent map[string]inventory.Send - lastSentUnread string } diff --git a/cmd/mesh-controller/build_source.go b/cmd/mesh-controller/build_source.go index 619aef0c..85f105c6 100644 --- a/cmd/mesh-controller/build_source.go +++ b/cmd/mesh-controller/build_source.go @@ -524,7 +524,6 @@ type registrations interface { SourceIdentity(ctx context.Context, module string) (int64, error) Catalogued(ctx context.Context) ([]inventory.Entry, error) BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error) - MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error) } // mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in. diff --git a/cmd/mesh-controller/build_source_integration_test.go b/cmd/mesh-controller/build_source_integration_test.go index 66645552..5ae12b9b 100644 --- a/cmd/mesh-controller/build_source_integration_test.go +++ b/cmd/mesh-controller/build_source_integration_test.go @@ -129,41 +129,6 @@ func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) { } } -func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - inv := open.inventory - fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "evil"}) - fork.Commit = "c0ffee0123456789" // the commit sudo was registered at - keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false) - if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) { - t.Fatalf("the fork's build was taken in: %v", err) - } - // Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older - // than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main. - failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "2"}) - failed.Commit = "badbadbad0123456" - if _, _, err := takeIn(ctx, inv, failed); err != nil { - t.Fatal(err) - } - record, _, err := inv.BuildByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record) - if err != nil { - t.Fatal(err) - } - if found && previous.ID == fork.ID { - t.Fatalf("a rollback would put back the fork's build %s", previous.ID) - } - if !found || previous.ID != "build-sudo" { - t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found) - } -} - func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) { open := theCatalogue(t) ctx := t.Context() @@ -305,10 +270,11 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) { t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { - t.Fatal(err) - } + var forgeMerge keptMerge + _ = json.Unmarshal(event, &forgeMerge) + wasForge := keptForgeMerge + t.Cleanup(func() { keptForgeMerge = wasForge }) + keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil } id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git", Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil { diff --git a/cmd/mesh-controller/builder_on_the_bus_test.go b/cmd/mesh-controller/builder_on_the_bus_test.go index 4ab796b6..8c6afb64 100644 --- a/cmd/mesh-controller/builder_on_the_bus_test.go +++ b/cmd/mesh-controller/builder_on_the_bus_test.go @@ -47,6 +47,11 @@ func aBuilderOnTheBus(t *testing.T) builderOnTheBus { } *c.into = kv } + // And the controller's own buckets, as its start asserts them: a merge it hears is kept in the bucket of the merges + // (ADR 0539 §17), which no other test need have made on this bus first. + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } servingBus.Store(js) t.Cleanup(func() { servingBus.Store(nil) }) withTrunkList(t, true) diff --git a/cmd/mesh-controller/bus_runs_test.go b/cmd/mesh-controller/bus_runs_test.go new file mode 100644 index 00000000..7c77c6ba --- /dev/null +++ b/cmd/mesh-controller/bus_runs_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + "testing" +) + +// 1945 (dev-r6-controller's B8 follows it): what a bus machine runs is read from its have, not from a send kept, so a +// machine whose have names the build the mesh holds is not moved by a bus upgrade, and one whose have names an older +// build is. +func TestABusMachineRunsWhatItsHaveNames(t *testing.T) { + was := busRuns + t.Cleanup(func() { busRuns = was }) + busRuns = func(_ context.Context, node, _ string) (string, bool, error) { + return map[string]string{"novox": "src2:new", "ace": "src2:old"}[node], node != "g14", nil + } + b := busPending{module: "nats", to: "c-new", from: map[string]string{}, same: map[string]bool{}} + commitOf := map[string]string{"src2:new": "c-new", "src2:old": "c-old"} + for _, n := range []string{"novox", "ace", "g14"} { + fp, known, _ := busRuns(context.Background(), n, "nats") + if c, ok := commitOf[fp]; known && ok { + b.from[n] = c + } + } + if b.moves("novox") || !b.moves("ace") || !b.moves("g14") { + t.Errorf("novox moves %v (runs the new build), ace %v (an older one), g14 %v (says none)", b.moves("novox"), + b.moves("ace"), b.moves("g14")) + } +} diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index e52f97ce..0d1e3202 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -10,8 +10,10 @@ import ( "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -111,20 +113,44 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro b.same[commit] = true } } + // What each machine runs is its have's fingerprint (novox/hq ADR 0539 clause 9), the commit of the build of it. + commitOf := map[string]string{} + for commit, src := range sources { + if src != "" { + commitOf[src] = commit + } + } for _, n := range b.machines { - sent, known, err := inv.SentBuilds(ctx, n) + fp, known, err := busRuns(ctx, n, b.module) if err != nil { return b, err } - if known { - if c, carried := sent[b.module]; carried { - b.from[n] = c - } + if c, ok := commitOf[fp]; known && ok { + b.from[n] = c } } return b, nil } +// busRuns is the fingerprint a machine's node-engine says it runs of a module, from its have; known is false when it +// says none. A variable so a test reads it without a bus. +var busRuns = func(ctx context.Context, node, module string) (string, bool, error) { + bus, err := aBus() + if err != nil { + return "", false, err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return "", false, err + } + h, found, err := haveOn(ctx, js, node, module) + if err != nil || !found || h.FP == "" { + return "", false, err + } + return h.FP, true, nil +} + // busHeld names the machines a push may not send because sending them would replace the bus: the // planned step's, not a push's (ADR 0236). Said with the remedy. func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) { @@ -229,17 +255,85 @@ func busCommand(ctx context.Context, args []string) error { fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From, short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it", false: "NOT reversible: the snapshot is the only way back"}[*reversible]) - sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving) + // **Nothing is sent, and the assigner never moves the bus** (novox/hq ADR 0539 clause 15): the bus's machines are + // composed at the build the mesh holds, and the operator writes each machine's want with the operator's credential, + // recorded, naming that composition. + wants, err := busWantsToWrite(withBusStep(ctx), open, b.module, moving) if err != nil { - _ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error()) - return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err) + _ = inv.EndBusStep(ctx, step.ID, "failed", "the bus's composition could not be written: "+err.Error()) + return fmt.Errorf("the bus's machines could not be composed at its new build: %w — nothing was replaced", err) + } + fmt.Printf("composed %s at the bus's new build. Write each want below with the operator's credential, and record it "+ + "in the hand-act log; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+ + "consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot as the "+ + "way back. `bus` says how it went\n", strings.Join(moving, ", "), busStepBound) + for _, w := range wants { + fmt.Println(" " + w) } - fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+ - "consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+ - "as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound) return nil } +// busWantsToWrite composes the bus's machines and answers, for each, the want the operator writes: its key in the +// assigner's bucket and its value, naming the fingerprint the bus module is registered at and that machine's +// composition of it. +func busWantsToWrite(ctx context.Context, open *stores, module string, machines []string) ([]string, error) { + if _, err := composeModules(ctx, open, machines); err != nil { + return nil, err + } + fps, err := currentFingerprints(ctx, open.inventory) + if err != nil { + return nil, err + } + bus, err := aBus() + if err != nil { + return nil, err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return nil, err + } + digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket) + if err != nil { + return nil, err + } + digestOf := func(node string) (string, error) { + key, err := sdkdelivery.ComposedKey(node, module, fps[module]) + if err != nil { + return "", err + } + e, err := digests.Get(ctx, key) + if err != nil { + return "", fmt.Errorf("%s: %w", key, err) + } + v, err := sdkdelivery.DecodeComposedDigest(e.Value()) + return v.Digest, err + } + return busWants(module, fps[module], machines, digestOf) +} + +// busWants is each machine's want for the bus module, as the operator writes it: ` `. +func busWants(module, fp string, machines []string, digestOf func(node string) (string, error)) ([]string, error) { + var out []string + for _, node := range machines { + digest, err := digestOf(node) + if err != nil { + return nil, err + } + key, err := sdkdelivery.WantKey(node, module) + if err != nil { + return nil, err + } + value, err := sdkdelivery.EncodeWant(sdkdelivery.Want{FP: fp, Composed: digest, + Words: "the bus moved by a person's step (bus upgrade, ADR 0539 clause 15)"}) + if err != nil { + return nil, err + } + out = append(out, fmt.Sprintf("%s %s %s", sdkdelivery.AssignerWantsBucket, key, value)) + } + return out, nil +} + // busStatus is `bus`: what an upgrade would do, and the last step. func busStatus(ctx context.Context) error { open, err := openStores(ctx) @@ -297,7 +391,7 @@ func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, err if err != nil { return nil, err } - reports, err := inv.LastReports(ctx) + reports, err := lastReports(ctx, inv) if err != nil { return nil, err } diff --git a/cmd/mesh-controller/bus_step_wants_test.go b/cmd/mesh-controller/bus_step_wants_test.go new file mode 100644 index 00000000..77647c8d --- /dev/null +++ b/cmd/mesh-controller/bus_step_wants_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1945/AC7: a bus upgrade answers, for each machine, the want the operator writes: the assigner's bucket, the +// machine's key for the bus module, and a want naming the bus's fingerprint and that machine's composition of it. It +// sends nothing: nothing it calls publishes on a node's declaration subject (busWants composes no send at all). +func TestABusUpgradeAnswersTheWantTheOperatorWrites(t *testing.T) { + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64) + digests := map[string]string{"novox": "sha256:" + strings.Repeat("1", 64), "ace": "sha256:" + strings.Repeat("2", 64)} + lines, err := busWants("nats", fp, []string{"ace", "novox"}, func(node string) (string, error) { return digests[node], nil }) + if err != nil || len(lines) != 2 { + t.Fatalf("busWants = %v, %v", lines, err) + } + for i, node := range []string{"ace", "novox"} { + parts := strings.SplitN(lines[i], " ", 3) + if len(parts) != 3 || parts[0] != sdkdelivery.AssignerWantsBucket || parts[1] != "want."+node+".nats" { + t.Fatalf("line %q", lines[i]) + } + w, err := sdkdelivery.DecodeWant([]byte(parts[2])) + if err != nil || w.FP != fp || w.Composed != digests[node] { + t.Errorf("%s's want = %+v, %v", node, w, err) + } + } + if _, err := busWants("nats", fp, []string{"ace"}, func(string) (string, error) { return "", nil }); err == nil { + t.Errorf("a want naming no composition was answered") + } +} diff --git a/cmd/mesh-controller/bus_waits.go b/cmd/mesh-controller/bus_waits.go deleted file mode 100644 index 0c02e870..00000000 --- a/cmd/mesh-controller/bus_waits.go +++ /dev/null @@ -1,195 +0,0 @@ -package main - -import ( - "fmt" - "sort" - "strings" - "sync" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A send held for the bus's planned step (novox/hq issue 336). -// -// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned -// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every -// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only -// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move. -// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what -// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it -// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the -// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks' -// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB). - -// kindBusStepWaiting is S17's kind: bus..step-waiting. -const kindBusStepWaiting = "bus-step-waiting" - -// busFacts is a new bus build waiting for its planned step, and the sends held for it. -type busFacts struct { - module, to string - // from is the build each machine of the bus runs; machines those whose bus the step would replace. - from map[string]string - machines []string - waits []busWaitFacts -} - -// busWaitFacts is one walk whose send was refused because it would replace the bus. -type busWaitFacts struct { - plan, repository, commit string - // modules are what its tier sends: what waits. - modules []string - // since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal. - since time.Time -} - -// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note -// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save. -var busRefusedFirst = &firstSeen{at: map[string]time.Time{}} - -type firstSeen struct { - mu sync.Mutex - at map[string]time.Time -} - -// mark keeps the first moment an id was seen. -func (s *firstSeen) mark(id string, at time.Time) { - s.mu.Lock() - defer s.mu.Unlock() - if _, seen := s.at[id]; !seen { - s.at[id] = at - } -} - -// of is when an id was first seen; zero when it was not. -func (s *firstSeen) of(id string) time.Time { - s.mu.Lock() - defer s.mu.Unlock() - return s.at[id] -} - -// keepOnly forgets every id not given: a walk no longer held is not held since then. -func (s *firstSeen) keepOnly(ids map[string]bool) { - s.mu.Lock() - defer s.mu.Unlock() - for id := range s.at { - if !ids[id] { - delete(s.at, id) - } - } -} - -// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step. -func refusedForTheBus(err error) bool { - return err != nil && strings.Contains(err.Error(), errBusWaits.Error()) -} - -// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its -// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace -// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not. -func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts { - f := busFacts{module: b.module, to: b.to, from: b.from} - for _, n := range b.machines { - if b.moves(n) { - f.machines = append(f.machines, n) - } - } - if len(f.machines) == 0 { - return f - } - for _, p := range plans { - if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) { - continue - } - since := p.Updated - if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) { - since = seen - } - var modules []string - if p.Tier >= 0 && p.Tier < len(p.Tiers) { - modules = append(modules, p.Tiers[p.Tier]...) - } else { - modules = planModules(p) - } - sort.Strings(modules) - f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules, - since: since}) - } - sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) }) - return f -} - -// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out. -func (b busFacts) heldByTheBus() map[string]bool { - out := map[string]bool{} - for _, w := range b.waits { - out[w.plan] = true - } - return out -} - -// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with -// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true"). -const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`" - -// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator. -func watchBusWaits(f *signalFacts) []conditions.Observation { - b := f.bus - if len(b.waits) == 0 || len(b.machines) == 0 { - return nil - } - since := b.waits[0].since - var walks, what []string - for _, w := range b.waits { - walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit), - strings.Join(w.modules, ", "))) - what = append(what, deliveryWhat(w.modules, w.repository)) - } - var from []string - for _, n := range b.machines { - from = append(from, short(orNotKnown(b.from[n]))) - } - machines := namesWords(b.machines, 3) - in := f.now.Sub(since) - return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting", - Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, - Machine: b.machines[0], Also: b.machines[1:], - Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+ - "the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "), - b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb, - since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")), - Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)), - Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax), - Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+ - "installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines, - namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax), - Needs: "start the bus upgrade " + FromMeshMCPServer, - Resolved: "Resolved: the bus upgrade started, and sends go on"}} -} - -// sortedUnique is a list sorted, each once. -func sortedUnique(xs []string) []string { - seen := map[string]bool{} - var out []string - for _, x := range xs { - if !seen[x] { - seen[x] = true - out = append(out, x) - } - } - sort.Strings(out) - return out -} - -// clipWords keeps a plain sentence within a bound, at a word. -func clipWords(s string, n int) string { - if len(s) <= n { - return s - } - cut := strings.LastIndex(s[:n-1], " ") - if cut <= 0 { - cut = n - 1 - } - return s[:cut] + "…" -} diff --git a/cmd/mesh-controller/bus_waits_integration_test.go b/cmd/mesh-controller/bus_waits_integration_test.go deleted file mode 100644 index 6fec986c..00000000 --- a/cmd/mesh-controller/bus_waits_integration_test.go +++ /dev/null @@ -1,70 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) { - now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) - b := aBusOnAnchor() - walk := func(id, repo, note string, updated time.Time) inventory.Plan { - return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0, - Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note, - Updated: updated} - } - for _, c := range []struct { - name string - bus busPending - plans []inventory.Plan - first map[string]time.Time - want []string - since time.Time - }{ - {"a walk refused for the bus", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil, - []string{"plan-1"}, now.Add(-27 * time.Minute)}, - {"refused earlier than its last save, as this controller saw it", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))}, - map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)}, - {"a walk refused for another reason", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil, - nil, time.Time{}}, - {"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(), - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"the bus already runs the build held: its step started", func() busPending { - o := aBusOnAnchor() - o.from = map[string]string{"anchor": o.to} - return o - }(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan { - p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now) - p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"} - return []inventory.Plan{p} - }(), nil, nil, time.Time{}}, - } { - t.Run(c.name, func(t *testing.T) { - got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] }) - var ids []string - for _, w := range got.waits { - ids = append(ids, w.plan) - } - if strings.Join(ids, ",") != strings.Join(c.want, ",") { - t.Fatalf("held for the bus: %v, want %v", ids, c.want) - } - if len(c.want) > 0 { - if !got.waits[0].since.Equal(c.since) { - t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since) - } - if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" { - t.Errorf("behind %+v", got) - } - } - }) - } -} diff --git a/cmd/mesh-controller/bus_waits_test.go b/cmd/mesh-controller/bus_waits_test.go deleted file mode 100644 index 668d7cff..00000000 --- a/cmd/mesh-controller/bus_waits_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a -// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that -// ends it — and the wait is not read as a walk running late. - -// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held. -func aBusOnAnchor() busPending { - return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"}, - to: "32307bd1bbbb", same: map[string]bool{}} -} - -// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it. -func refusedNote(b busPending, tier int) string { - held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) + - "), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)" - return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again" -} - -// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises -// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to, -// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the -// condition clears once the bus's machine runs the new build. -func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) { - now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) - b := aBusOnAnchor() - held := func(entered time.Duration) *signalFacts { - f := calm(now) - f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1, - entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}} - f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"}, - waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122", - modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}} - return f - } - - f := held(time.Minute) - got := watchBusWaits(f) - if len(got) != 1 { - t.Fatalf("a send held for the bus a minute ago raised %+v", got) - } - o := got[0] - if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator { - t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver) - } - for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus", - `"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("its summary does not say %q: %s", want, o.Summary) - } - } - if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" { - t.Errorf("its words do not say what waits and what the operator does: %+v", o) - } - if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, - Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok { - t.Errorf("its words are not plain: %s", why) - } - - // Past S3's bound: still the bus's wait, never a stalled walk. - late := held(45 * time.Minute) - if s3 := watchPlans(late); len(s3) != 0 { - t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3) - } - // A walk held for something else past its bound is still stalled. - other := held(45 * time.Minute) - other.plans[0].bus = false - if s3 := watchPlans(other); len(s3) != 1 { - t.Fatalf("a walk held for something else past its bound raised %+v", s3) - } - - // Through the keeper: open at the first tick, cleared when the step started. - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}, - Now: func() time.Time { return now }}) - defer k.Close(context.Background()) - w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} - w.see(t.Context(), held(time.Minute)) - open, err := k.Open(t.Context()) - if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" { - t.Fatalf("after the first tick, open: %+v (%v)", open, err) - } - started := held(time.Minute) - started.bus = busFacts{module: "nats", to: b.to} - started.plans[0].bus = false - w.see(t.Context(), started) - if open, _ := k.Open(t.Context()); len(open) != 0 { - t.Fatalf("the step started, and open: %+v", open) - } -} diff --git a/cmd/mesh-controller/busobjects.go b/cmd/mesh-controller/busobjects.go index f7213d32..701692bc 100644 --- a/cmd/mesh-controller/busobjects.go +++ b/cmd/mesh-controller/busobjects.go @@ -29,7 +29,13 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra for _, n := range nodes { names = append(names, n.Name) } - if err := broker.Raise(r, names); err != nil { + // Once every node runs the node-engine that reads no declaration, the declarations' stream and the nodes' + // consumers on it are not asserted again (novox/hq ADR 0539 clause 11; composed_start.go retires them). + raise := func() error { return broker.Raise(r, names) } + if declarationsRetiredNow(ctx, inv) { + raise = func() error { return broker.RaiseAfterDeclarations(r) } + } + if err := raise(); err != nil { return nil, err } // And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own diff --git a/cmd/mesh-controller/busobjects_test.go b/cmd/mesh-controller/busobjects_test.go index 53a9145e..cd3b0b69 100644 --- a/cmd/mesh-controller/busobjects_test.go +++ b/cmd/mesh-controller/busobjects_test.go @@ -7,14 +7,9 @@ import ( "strings" "testing" - "github.com/nats-io/nats.go" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not @@ -165,43 +160,3 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) { t.Fatalf("a send that asserted everything left the condition open: %v", err) } } - -// Against a real bus, through the send's own grant: a module assigned after start has its consumer -// once a declaration is sent, and a holder assigned after start its seat's worker. -func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - // The controller starting, before either was assigned. - if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil { - t.Fatal(err) - } - consumer := aCarriedConsumer(t, open) - worker := aLateHolder(t, open) - _ = js.Context().DeleteConsumer(worker.Stream, worker.Name) - for _, c := range []broker.Consumer{consumer, worker} { - if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) { - t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err) - } - } - - server := link.ConnectNats(js, nil, nil) - if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil { - t.Fatal(err) - } - for _, c := range []broker.Consumer{consumer, worker} { - if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil { - t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err) - } - } - if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised { - t.Fatal("a send that asserted everything raised a condition") - } -} diff --git a/cmd/mesh-controller/busy.go b/cmd/mesh-controller/busy.go deleted file mode 100644 index 960a1bab..00000000 --- a/cmd/mesh-controller/busy.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "fmt" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A first-node gate waits for a busy container (novox/hq ADR 0335 §4). -// -// When a machine reports, on what the gate sent it, that a container of a judged module was left running -// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it -// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its -// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded -// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped -// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and -// its gate starts as before. - -// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the -// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here. -var busyGateSlack = 5 * time.Minute - -// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this -// controller's clock from when it first read the wait. -const busyEngineBound = 15 * time.Minute - -// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of -// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the -// machine nor a wrong word from it holds a walk longer. -func busyGateEnds(b *inventory.GateBusy) time.Time { - until := b.Seen.Add(busyEngineBound) - if !b.Until.IsZero() && b.Until.Before(until) { - until = b.Until - } - return until.Add(busyGateSlack) -} - -// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges: -// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the -// gate's Since to its end. Pure but for the gate it is handed. -func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool { - if g.Busy != nil && g.Busy.Ended != nil { - return false - } - for _, j := range pairs { - b, waits := busyOn(j, f) - if !waits { - continue - } - if g.Busy == nil { - g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since, - Seen: now} - } - if g.Busy.Seen.IsZero() { - g.Busy.Seen = now - } - g.Busy.Until, g.Busy.Said = b.Until, b.Said - if now.After(busyGateEnds(g.Busy)) { - // The engine said it would have replaced it by now: the judging starts, and finds the module not - // yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound. - end := now - g.Busy.Ended, g.Busy.PastBound = &end, true - g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil - return false - } - g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil - g.Read(now, false, g.Busy.Words()) - return true - } - if g.Busy == nil { - return false - } - // Replaced: the judging starts at the report that said so. - end := now - if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) { - end = r.At.UTC() - } - g.Busy.Ended = &end - g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil - return false -} - -// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left -// running because it said busy. -func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) { - r, said := f.reports[j.node] - if !said || r.At == nil || !f.reportedOn(j.node, r) { - return inventory.BusyWait{}, false - } - return inventory.WaitingFor(r.Busy, j.module) -} - -// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting -// for it: not yet what was sent, so not healthy, whatever its health says of the container still running. -func busyStill(j judged, f gateFacts) (string, bool) { - b, waits := busyOn(j, f) - if !waits { - return "", false - } - return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s", - j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true -} diff --git a/cmd/mesh-controller/busy_test.go b/cmd/mesh-controller/busy_test.go deleted file mode 100644 index 4b8433e5..00000000 --- a/cmd/mesh-controller/busy_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk whose module's container waits as busy on its first machine says so, and starts that machine's -// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4). -func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) { - sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent, - Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}} - pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}} - reported := sent.Add(20 * time.Second) - waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute), - Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}} - facts := func(at time.Time, busy []inventory.BusyWait) gateFacts { - // As judgeMoves hands it: the report held against the gate's own send. - return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox", - Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}} - } - - now := reported.Add(time.Minute) - if !busyWait(g, pairs, facts(reported, waiting), now) { - t.Fatal("a gate whose module's container waits as busy judged it") - } - if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil { - t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy) - } - for _, said := range []string{gateLine(g), g.Last} { - if !strings.Contains(said, "waits for a busy container on novox") { - t.Errorf("the walk does not say it waits for a busy container there: %s", said) - } - } - if !strings.Contains(g.Last, "check of #280") { - t.Errorf("the wait does not say what the container holds: %s", g.Last) - } - - // Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run. - now = reported.Add(10 * time.Minute) - if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 { - t.Fatalf("the gate stopped waiting within the bound: %+v", g) - } - - // The machine reports it replaced: the judging starts at that report. - replacedAt := reported.Add(12 * time.Minute) - now = replacedAt.Add(5 * time.Second) - replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}} - if busyWait(g, pairs, facts(replacedAt, replaced), now) { - t.Fatal("the gate still waits after the container was replaced") - } - if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) { - t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy) - } - if strings.Contains(gateLine(g), "waits for a busy container") { - t.Errorf("a gate judging still says it waits: %s", gateLine(g)) - } - // And a later report listing it waiting again does not hold a gate that started. - if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) { - t.Error("a gate that started waits again") - } -} - -// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack, -// the judging starts, and finds the module not yet healthy while its container is still listed waiting. -func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) { - sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} - pairs := []judged{{module: "build-agent", node: "novox"}} - waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}} - f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied, - At: &sent, Current: true, Busy: waiting}}} - now := sent.Add(15*time.Minute + busyGateSlack + time.Second) - if busyWait(g, pairs, f, now) { - t.Fatal("a gate waited past the engine's bound and the slack") - } - if !g.Busy.PastBound || !g.Since.Equal(now) { - t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since) - } - if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") { - t.Fatalf("a module still running its busy container reads as what was sent: %q", why) - } -} - -// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a -// report not about the gate's send says nothing of it. -func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) { - sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent, - Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}} - pairs := []judged{{module: "build-agent", node: "ace"}} - at := sent.Add(time.Minute) - older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied, - At: &at, Declared: "d2", ReportedSequence: 9}}} - if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) { - t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since) - } - elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied, - At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server", - State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}} - if busyWait(g, pairs, elsewhere, at) { - t.Fatal("a wait reported about an earlier declaration held the gate") - } -} - -// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until -// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait. -func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) { - seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen} - pairs := []judged{{module: "build-agent", node: "novox"}} - ahead := seen.Add(time.Hour) - f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied, - At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}} - if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) { - t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy) - } - if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) { - t.Fatal("the gate stopped waiting within the bound on its own clock") - } - if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound { - t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy) - } -} - -// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent. -func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) { - modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{ - {"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}}, - {"id": "agent-env", "type": "file"}}}} - lines := busyNotSent(modules, false) - if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") || - !strings.Contains(lines[0], "replaced at once") { - t.Fatalf("the plan does not say the busy not sent: %v", lines) - } - if lines := busyNotSent(modules, true); len(lines) != 0 { - t.Fatalf("a busy sent is said as not sent: %v", lines) - } -} diff --git a/cmd/mesh-controller/calls_verb_test.go b/cmd/mesh-controller/calls_verb_test.go index 4ff06dcb..d0958017 100644 --- a/cmd/mesh-controller/calls_verb_test.go +++ b/cmd/mesh-controller/calls_verb_test.go @@ -24,28 +24,4 @@ func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) { } } -// A push — named or through command — answers before it runs: it sends the machine holding the bus -// first, and the broker reloading its user list forgets the answer it was about to permit. -func TestAPushAnswersBeforeItSends(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - want bool - }{ - {"push", map[string]any{"node": "anchor", "why": "w"}, true}, - {"push", map[string]any{"why": "w"}, true}, - {"command", map[string]any{"command": "builds"}, false}, - {"status", map[string]any{}, false}, - {"assign", map[string]any{"node": "anchor", "module": "m"}, false}, - } { - argv, err := argvFor(c.verb, c.args) - if err != nil { - t.Fatalf("%s %v: %v", c.verb, c.args, err) - } - if got := answersFirst(argv); got != c.want { - t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want) - } - } -} - // `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call. diff --git a/cmd/mesh-controller/catch_up.go b/cmd/mesh-controller/catch_up.go deleted file mode 100644 index f69e4afc..00000000 --- a/cmd/mesh-controller/catch_up.go +++ /dev/null @@ -1,643 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "net/url" - "path" - "regexp" - "slices" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/asks" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **The controller catches up on merges it did not hear** (novox/hq ADR 0457, issue 1336). A merge the forge announced while the -// controller was being replaced was never heard, and nothing asked for it again: the trunk held a merge no walk answered. At its -// start, and each time it hears a merge, the controller reads the last catchUpWindow merge commits of a trunk from the forge, and -// hears each one it holds no record of as if the forge had announced it then, on the warrant the router holds for it. -// It lasts until release 6 of design 47 deletes delivery plans and batches (ADR 0429 §7, §8). - -// catchUpWindow is how many of a trunk's newest merge commits are read; an older merge is a person's to decide. -const catchUpWindow = 50 - -// kindMergeNotHeard is the condition of a merge the controller did not hear and holds no warrant to plan. -const kindMergeNotHeard = "merge-not-heard" - -// trunkMerge is a merge commit of a trunk as the forge lists it. -type trunkMerge struct { - Commit string - // Head is the merge commit's second parent: the pull request's head that was merged. - Head string - Subject string - At time.Time -} - -// trunkCommit is a commit of a trunk as the forge lists it, newest first: a merge commit has a second parent. -type trunkCommit struct { - Commit, SecondParent, Subject string - At time.Time -} - -// trunkCommitsPage is one page (fifty) of a trunk's commits, newest first; empty past the last. A variable so a test needs no -// forge. -var trunkCommitsPage = func(ctx context.Context, owner, repo, branch string, page int) ([]trunkCommit, error) { - var found struct { - Result []struct { - SHA string `json:"sha"` - Parents []struct { - SHA string `json:"sha"` - } `json:"parents"` - Commit struct { - Message string `json:"message"` - Committer struct { - Date string `json:"date"` - } `json:"committer"` - } `json:"commit"` - } `json:"result"` - } - apiPath := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&limit=50&page=%d&stat=false&verification=false&files=false", - owner, repo, url.QueryEscape(branch), page) - missing, err := askForgeApi(ctx, apiPath, &found) - if err != nil { - return nil, err - } - if missing { - return nil, fmt.Errorf("the forge holds no branch %s of %s/%s", branch, owner, repo) - } - var out []trunkCommit - for _, c := range found.Result { - at, _ := time.Parse(time.RFC3339, c.Commit.Committer.Date) - tc := trunkCommit{Commit: c.SHA, Subject: firstLine(c.Commit.Message), At: at.UTC()} - if len(c.Parents) >= 2 { - tc.SecondParent = c.Parents[1].SHA - } - out = append(out, tc) - } - return out, nil -} - -// readTrunkMerges is the last catchUpWindow merge commits of a repository's trunk, oldest first: pages are read until that many -// merges are held or the trunk ends. -func readTrunkMerges(ctx context.Context, owner, repo, branch string) ([]trunkMerge, error) { - var merges []trunkMerge - for page := 1; page <= 8 && len(merges) < catchUpWindow; page++ { - commits, err := trunkCommitsPage(ctx, owner, repo, branch, page) - if err != nil { - return nil, err - } - if len(commits) == 0 { - break - } - for _, c := range commits { - // A merge of the trunk is a pull request's merge into it. A commit with a second parent that is none is a merge made - // inside some branch ("Merge remote-tracking branch 'origin/main' into fix/..."), which the trunk's history lists too and - // which no Approve ever covered: read as a gap it would raise a condition for nothing a person decides. - if c.SecondParent != "" && mergesIntoTrunk(c.Subject, branch) { - merges = append(merges, trunkMerge{Commit: c.Commit, Head: c.SecondParent, Subject: c.Subject, At: c.At}) - } - } - } - return lastMerges(merges), nil -} - -// sinceTheRecord are the merges that can be gaps (novox/hq ADR 0463 §2): those made since the controller's record of merges begins on -// the mesh (first, the earliest merge it holds of any repository and branch, less mergeGrace as owedLate judges). A merge before it is -// history on every trunk, and a trunk with no merge kept of its own still has the merges made since as gaps. With no merge kept -// anywhere there is no record to have a gap in. -func sinceTheRecord(merges []trunkMerge, kept bool, first time.Time) []trunkMerge { - if !kept { - return nil - } - var out []trunkMerge - for _, m := range merges { - if !m.At.Before(first.Add(-mergeGrace)) { - out = append(out, m) - } - } - return out -} - -// lastMerges is the newest catchUpWindow of merges, given newest first, oldest first. -func lastMerges(newestFirst []trunkMerge) []trunkMerge { - if len(newestFirst) > catchUpWindow { - newestFirst = newestFirst[:catchUpWindow] - } - out := slices.Clone(newestFirst) - slices.Reverse(out) - return out -} - -var mergeSubject = regexp.MustCompile(`^Merge pull request '(.*)' \(#(\d+)\) from (\S+) into (\S+)$`) - -// pullOfSubject is the pull request a merge commit's subject names: its title and number. -func pullOfSubject(subject string) (title string, number int, ok bool) { - m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)) - if m == nil { - return "", 0, false - } - n, err := strconv.Atoi(m[2]) - if err != nil || n <= 0 { - return "", 0, false - } - return m[1], n, true -} - -// mergesIntoTrunk says a commit's subject is a pull request's merge into this trunk. -func mergesIntoTrunk(subject, branch string) bool { - m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)) - return m != nil && m[4] == branch -} - -// headBranchOfSubject is the branch a merge commit's subject says was merged. -func headBranchOfSubject(subject string) string { - if m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)); m != nil { - return m[3] - } - return "" -} - -// readMergeEvent is the merge as the forge would have announced it: the commit, the pull request, the files it changed and the -// directories among them that hold a module.json. A variable so a test needs no forge. -var readMergeEvent = func(ctx context.Context, owner, repo, branch string, tm trunkMerge) (link.SourceMoved, error) { - m := link.SourceMoved{Owner: owner, Repo: repo, Base: branch, Head: headBranchOfSubject(tm.Subject), HeadSHA: tm.Head, - Commit: tm.Commit, MergedAt: tm.At.UTC().Format(time.RFC3339)} - if title, number, ok := pullOfSubject(tm.Subject); ok { - m.Number, m.Title = number, title - } - var found struct { - Result struct { - Files []struct { - Filename string `json:"filename"` - Status string `json:"status"` - } `json:"files"` - } `json:"result"` - } - missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(tm.Commit), &found) - if err != nil { - return m, err - } - if missing || len(found.Result.Files) == 0 { - return m, fmt.Errorf("the forge named no files for the merge %.8s of %s/%s", tm.Commit, owner, repo) - } - for _, f := range found.Result.Files { - m.Paths = append(m.Paths, f.Filename) - if f.Status == "removed" || f.Status == "deleted" { - m.Removed = append(m.Removed, f.Filename) - } - } - // The forge lists a merge's files by the page; a list as long as a page may be a beginning. - m.PathsTruncated = len(m.Paths) >= 300 - dirs := map[string]bool{} - for _, p := range m.Paths { - for d := path.Dir(p); ; d = path.Dir(d) { - dirs[d] = true - if d == "." || d == "/" { - break - } - } - } - if len(dirs) > 64 { - return m, nil // too many to look at: the old rule stands, a directory is a module when its manifest changed - } - var holding []string - for d := range dirs { - file := "module.json" - if d != "." { - file = d + "/module.json" - } - _, exists, err := manifestCached(ctx, owner, repo, file, tm.Commit) - if err != nil { - return m, err - } - if exists { - holding = append(holding, d) - } - } - sort.Strings(holding) - m.ModuleDirs, m.ModuleDirsSaid = holding, true - return m, nil -} - -// mergeAskPrefix is what the id of a merge ask made by the retired module mesh-delivery begins with: the pull request's number -// and the head's first twelve digits (`merge---`). The teller's ids name neither (see findMergeWarrant). -func mergeAskPrefix(number int, head string) string { - if len(head) > 12 { - head = head[:12] - } - return fmt.Sprintf("merge-%d-%s-", number, head) -} - -// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking -// mesh-delivery's (ADR 0450 §8; issue 1318). -func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} } - -// caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant -// answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437 -// §6, as movedOnApprove judges an announced one). The heads the Approve was bound to are checked by the caller, where it can -// name them. -func caughtUpWarrant(asker string, ask *asks.Ask, w *asks.Warrant) (string, error) { - if err := askerAccepted(asker); err != nil { - return "", err - } - switch { - case ask == nil || w == nil: - return "", errors.New("the router holds no closed record of the ask") - case ask.Who != asks.Operator: - return "", fmt.Errorf("the ask %s is for %q to answer, not the operator", ask.ID, ask.Who) - } - option, err := w.For(asker, *ask) - if err != nil { - return "", err - } - if option.ID != optionApprove || w.Option != optionApprove { - return "", fmt.Errorf("the operator chose %q on the ask %s, not an Approve", w.Option, w.Ask) - } - if w.By == nil || w.By.Who != asks.Operator || w.By.Verified == "" { - return "", fmt.Errorf("the ask %s was not answered by the operator on a channel that proves who answered", w.Ask) - } - return w.Says() + " (ask " + w.Ask + ")", nil -} - -// routerAskIDs are the ids of the asks an asker has in the router's record. A variable so a test needs no bus. -var routerAskIDs = func(ctx context.Context, inv catalogueReader, asker string) ([]string, error) { - serving := servingBus.Load() - if serving == nil { - return nil, errors.New("only the serving controller reads the router's record") - } - bucket, err := asksRecords(ctx, inv) - if err != nil { - return nil, err - } - if bucket == "" { - return nil, errors.New("no module declares the operator channel's records") - } - api, err := jetstream.New(serving.Conn()) - if err != nil { - return nil, err - } - kv, err := api.KeyValue(ctx, bucket) - if err != nil { - return nil, err - } - return askIDsOf(ctx, kv, asker) -} - -// askIDsOf are the ids of the asks an asker has in the router's bucket. -func askIDsOf(ctx context.Context, kv jetstream.KeyValue, asker string) ([]string, error) { - lister, err := kv.ListKeysFiltered(ctx, asker+".>") - if err != nil { - return nil, err - } - var ids []string - for key := range lister.Keys() { - ids = append(ids, strings.TrimPrefix(key, asker+".")) - } - // The lister stops its own watch when it has sent the last key, so a stop of ours finds it stopped: nats answers that with - // "invalid subscription", which is no failure of the read (issue 1336). Any other error stays one. - if err := lister.Stop(); err != nil && !errors.Is(err, nats.ErrBadSubscription) { - return nil, err - } - return ids, nil -} - -// closedRecord is a router record of an ask that ended: it never changes, so it is read once. -type closedRecord struct { - ask *asks.Ask - w *asks.Warrant -} - -var closedRecords sync.Map // asker + "." + id -> closedRecord - -// closedRecordOf is the router's record of an ask, kept once it is closed. A nil ask is an ask still open, ended without a -// choice, or not there. -func closedRecordOf(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) { - if v, ok := closedRecords.Load(asker + "." + id); ok { - r := v.(closedRecord) - return r.ask, r.w, nil - } - ask, w, err := routerAskOf(ctx, inv, asker, id) - if err != nil { - return nil, nil, err - } - if ask != nil && w != nil { - closedRecords.Store(asker+"."+id, closedRecord{ask, w}) - } - return ask, w, nil -} - -// deliveryIDOf is the id the gate gives a delivery, and so the key of its head in the heads an Approve binds: the repository as -// owner/name, lower-cased, `@` and the first twelve characters of the head. -func deliveryIDOf(owner, repo, head string) string { - if len(head) > 12 { - head = head[:12] - } - return strings.ToLower(owner+"/"+repo) + "@" + head -} - -// coversHead says an ask the operator approved is for this merge: its Approve is bound to the heads of the pull request alone -// (mergeAct over {delivery id: head}, which is how a pull request not in a group is asked), or, for a group's ask, whose other -// members' heads the router's record does not carry in full, the ask as published names this pull request at this head in its -// details ("# ... at :"), and the digest the warrant answers covers those words. -func coversHead(ask *asks.Ask, owner, repo string, number int, head string) (how string, ok bool) { - option, found := ask.Option(optionApprove) - if !found { - return "", false - } - heads := map[string]string{deliveryIDOf(owner, repo, head): head} - if option.Performs(mergeAct(ask.ID, optionApprove, heads)) == nil { - return "bound to the head of this pull request", true - } - short := head - if len(short) > 12 { - short = short[:12] - } - line := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(repo) + `#` + strconv.Itoa(number) + ` .* at ` + regexp.QuoteMeta(short) + `:`) - if line.MatchString(ask.Details) { - return "a group's ask naming this pull request at this head", true - } - return "", false -} - -// findMergeWarrant finds the warrant of a merge, with no read the controller does not make today: the router's chosen merge -// ask for it. Under the teller (releases 3e on) the id is `merge---`, which names neither the pull -// request nor the head, so the teller's closed asks are read and the one the operator approved whose heads cover the merge's -// second parent is the warrant (coversHead). Under mesh-delivery, which made the earlier merges, the id names the pull request -// and the head and is found by them. Why it holds; or found false with why not, for a person to read; or an error when the -// router could not be read, which is no answer. A variable so a test needs no bus. -var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, repo string, number int, head string) (why string, found bool, reason string, err error) { - reason = "the router holds no chosen merge ask of " + strings.Join(askersOfMerges(), " or ") + " that covers the head " + head[:min(12, len(head))] - for _, asker := range askersOfMerges() { - ids, err := routerAskIDs(ctx, inv, asker) - if err != nil { - return "", false, "", err - } - for _, id := range ids { - named := number > 0 && strings.HasPrefix(id, mergeAskPrefix(number, head)) - if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") { - continue - } - ask, w, err := closedRecordOf(ctx, inv, asker, id) - if err != nil { - return "", false, "", err - } - if ask == nil || w == nil { - continue - } - warrant, err := caughtUpWarrant(asker, ask, w) - if err != nil { - if named { - reason = fmt.Sprintf("%s's ask %s holds no operator's Approve: %v", asker, id, err) - } - continue - } - how, covers := coversHead(ask, owner, repo, number, head) - if !covers && !named { - continue - } - if !covers { - how = "the ask's id names this pull request and head" - } - return warrant + ", " + how, true, "", nil - } - } - return "", false, reason, nil -} - -// caughtTrunk is one trunk the catalogue builds from. -type caughtTrunk struct{ Owner, Repo, Branch string } - -func (t caughtTrunk) repository() string { return t.Owner + "/" + t.Repo } - -// trunksOfCatalogue are the trunks of the repositories on the mesh's forge the catalogue builds a module from. -func trunksOfCatalogue(forms *sourceForms, entries []inventory.Entry) []caughtTrunk { - var out []caughtTrunk - for _, e := range entries { - if e.Provided || e.Source.Repository == "" { - continue - } - owner, name, onForge := forms.onTheForge(e.Source.Repository, e.Source.Seat) - if !onForge { - continue - } - branch := followedBranch(e.Source.Ref) - if branch == "" { - branch = "main" - } - t := caughtTrunk{owner, name, branch} - if !slices.Contains(out, t) { - out = append(out, t) - } - } - return out -} - -// catchUpBeside runs a catch-up beside the hearing of the merge that asked for it, so the bus's acknowledgement of that merge -// waits for no forge. A variable so a test runs it where it stands. -var catchUpBeside = func(run func()) { go run() } - -// catchingUp makes one catch-up at a time. -var catchingUp sync.Mutex - -// settledMerges are the merges this process found nothing to plan for: heard, and no record kept, because nothing reads them. -// They are not read from the forge again. -var settledMerges sync.Map - -// catchUp hears the merges of the catalogue's trunks (only one, when only names it) that the controller holds no record of. -// It says whether a trunk was left for later: the forge, the router or the store did not answer. -func (f following) catchUp(ctx context.Context, only *caughtTrunk) (later bool) { - catchingUp.Lock() - defer catchingUp.Unlock() - inv := f.open.inventory - entries, err := inv.Catalogued(ctx) - if err != nil { - fmt.Printf("catch-up: the catalogue cannot be read, so no merge is caught up now: %v\n", err) - return true - } - kept, first, err := inv.EarliestKept(ctx) - if err != nil { - fmt.Printf("catch-up: what the controller kept cannot be read, so no merge is caught up now: %v\n", err) - return true - } - if kept { - fmt.Printf("catch-up: the controller's record of merges begins %s; a merge made before it, less %s, is history on every trunk\n", - first.UTC().Format(time.RFC3339), mergeGrace) - } else { - fmt.Println("catch-up: the controller holds no merge of any trunk yet, so none can have been missed") - } - trunks := trunksOfCatalogue(newSourceForms(ctx, inv), entries) - if only != nil { - trunks = slices.DeleteFunc(trunks, func(t caughtTrunk) bool { - return !strings.EqualFold(t.repository(), only.repository()) || t.Branch != only.Branch - }) - } - for _, t := range trunks { - if f.catchUpTrunk(ctx, t, kept, first) { - later = true - } - } - return later -} - -func (f following) catchUpTrunk(ctx context.Context, t caughtTrunk, kept bool, first time.Time) (later bool) { - inv := f.open.inventory - repository := t.repository() - merges, err := readTrunkMerges(ctx, t.Owner, t.Repo, t.Branch) - if err != nil { - // Not nothing to catch up: left for the next start or the next heard merge (ADR 0457 clause 3). - fmt.Printf("catch-up: the forge did not answer for the last merges of %s %s, so they are not caught up now; the next start "+ - "or the next merge heard there does it: %v\n", repository, t.Branch, err) - return true - } - // **A merge made before the controller's record of merges begins is history, never a gap** (novox/hq ADR 0463 §2). - read := len(merges) - merges = sinceTheRecord(merges, kept, first) - fmt.Printf("catch-up: %s %s: %d of the last %d pull request merges were made since the record began\n", repository, t.Branch, - len(merges), read) - unwarranted := map[string]bool{} - for _, tm := range merges { - id := fmt.Sprintf("%s_%.8s", repository, tm.Commit) - if _, done := settledMerges.Load(repository + "@" + tm.Commit); done { - continue - } - if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err != nil { - fmt.Printf("catch-up: the record of %s cannot be read, so %s is not caught up now: %v\n", repository, tm.Commit[:8], err) - return true - } else if known { - continue - } - m, err := readMergeEvent(ctx, t.Owner, t.Repo, t.Branch, tm) - if err != nil { - fmt.Printf("catch-up: the merge %.8s of %s cannot be read from the forge, so it is not caught up now: %v\n", tm.Commit, repository, err) - later = true - continue - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return true - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return true - } - facts, err := factsOfMerge(ctx, inv, m, time.Now().UTC(), entries, read) - if err != nil { - fmt.Printf("catch-up: %s %.8s cannot be judged now: %v\n", repository, tm.Commit, err) - later = true - continue - } - // The very test hearMerge makes: a merge it would plan, or take as a first build, is a gap; any other it would say - // nothing reads, and is no gap for a person to decide. - if !facts.plans() && !facts.addsModuleToBuilds { - settledMerges.Store(repository+"@"+tm.Commit, true) - continue - } - why, found, reason, err := findMergeWarrant(ctx, inv, t.Owner, t.Repo, m.Number, tm.Head) - if err != nil { - fmt.Printf("catch-up: the router cannot be read, so %s %.8s is not caught up now: %v\n", repository, tm.Commit, err) - later = true - continue - } - if !found { - unwarranted[id] = true - f.raiseMergeNotHeard(ctx, repository, t.Branch, tm, m, reason) - continue - } - now := time.Now().UTC() - said := fmt.Sprintf("caught up: merged %s, not heard, planned %s", tm.At.Format(time.RFC3339), now.Format(time.RFC3339)) - fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard; %s; on the warrant %s\n", repository, t.Branch, tm.Commit, said, why) - m.Title = strings.TrimSpace(m.Title + " (" + said + ")") - if err := f.hearMerge(ctx, m, now); err != nil { - fmt.Printf("catch-up: %s %.8s could not be heard now: %v\n", repository, tm.Commit, err) - later = true - continue - } - if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err == nil && !known { - settledMerges.Store(repository+"@"+tm.Commit, true) // heard, and nothing reads it - } - } - // Cleared only when every gap was judged: one left for later may still be open. - if !later { - f.clearMergesHeard(ctx, repository, unwarranted) - } - return later -} - -// raiseMergeNotHeard keeps open, and observes again, the condition of a merge with no warrant to plan it. One for each -// repository and commit, however many catch-ups find it. -func (f following) raiseMergeNotHeard(ctx context.Context, repository, branch string, tm trunkMerge, m link.SourceMoved, reason string) { - fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard, and is not planned: %s\n", repository, branch, tm.Commit, reason) - if conditionsFrom == nil { - return - } - o := conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, tm.Commit), Kind: kindMergeNotHeard, - Severity: conditions.Warning, Source: "catch-up", Resolver: conditions.ResolverOperator, - Summary: fmt.Sprintf("the merge %.8s of %s into %s was not heard by the controller, and no operator's Approve of it is on "+ - "record to plan it on: a person decides", tm.Commit, repository, branch), - Said: fmt.Sprintf("%s %.8s (pull request %d, head %.8s, merged %s): %s", repository, tm.Commit, m.Number, tm.Head, - tm.At.Format(time.RFC3339), reason), - Headline: fmt.Sprintf("A merge into %s was never delivered", repository), - Explanation: fmt.Sprintf("A merge into %s was made while the controller did not hear it, and no Approve of yours for it is on "+ - "record that the controller can plan it on. It stays open while the merge is among the latest %d merges of that repository. "+ - "The details say what was looked for.", repository, catchUpWindow), - Needs: "decide whether that merge should be delivered, and say so if it should.", - Resolved: "the merge is planned, or is no longer among the latest merges of its repository"} - if _, err := conditionsFrom.Observe(ctx, o); err != nil { - fmt.Printf("catch-up: the condition of %s %.8s could not be kept: %v\n", repository, tm.Commit, err) - } -} - -// clearMergesHeard clears the open conditions of a repository's merges not heard that this catch-up did not find again: the -// gap was planned, or left the window. -func (f following) clearMergesHeard(ctx context.Context, repository string, still map[string]bool) { - if conditionsFrom == nil { - return - } - open, err := conditionsFrom.Open(ctx) - if err != nil { - fmt.Printf("catch-up: the open conditions cannot be read, so none is cleared now: %v\n", err) - return - } - for _, c := range open { - if c.Kind != kindMergeNotHeard || c.Subject.Scope != conditions.ScopeMerge || - !strings.HasPrefix(c.Subject.ID, repository+"_") || still[c.Subject.ID] { - continue - } - if _, err := conditionsFrom.Clear(ctx, c.Key, "the merge is planned, or left the last merges of its trunk"); err != nil { - fmt.Printf("catch-up: the condition %s could not be cleared: %v\n", c.Key, err) - } - } -} - -// catchUpAtStart catches up every trunk once the controller serves, and again every minute while a trunk was left for later. -func catchUpAtStart(ctx context.Context, open *stores) { - select { - case <-ctx.Done(): - return - case <-time.After(30 * time.Second): - } - for try := 0; try < 20; try++ { - if !(following{open: open}).catchUp(ctx, nil) { - return - } - select { - case <-ctx.Done(): - return - case <-time.After(time.Minute): - } - } - fmt.Println("catch-up: a trunk is still left for later after twenty tries; the next merge heard there does it") -} diff --git a/cmd/mesh-controller/catch_up_integration_test.go b/cmd/mesh-controller/catch_up_integration_test.go deleted file mode 100644 index c0b698fc..00000000 --- a/cmd/mesh-controller/catch_up_integration_test.go +++ /dev/null @@ -1,352 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear. - -// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the -// router holds an operator's Approve for. -type aTrunk struct { - merges []trunkMerge - module map[string]string - approved map[string]bool - read []string // the trunks asked of the forge, as owner/repo - events int // merges read in full from the forge - silent bool // the forge does not answer -} - -func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk { - t.Helper() - tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}} - for i, m := range modules { - c := fmt.Sprintf("%s-c%02d", repo, i+1) - tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1), - Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c), - At: t0.Add(time.Duration(i) * time.Minute)}) - tr.module[c] = m - tr.approved[c] = true - } - was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant - trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) { - tr.read = append(tr.read, owner+"/"+name) - if tr.silent { - return nil, errors.New("the forge did not answer") - } - if page > 1 || owner+"/"+name != "novox/"+repo { - return nil, nil - } - var newest []trunkCommit - for i := len(tr.merges) - 1; i >= 0; i-- { - m := tr.merges[i] - newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At}) - } - return newest, nil - } - readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) { - tr.events++ - m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At) - m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head - _, m.Number, _ = pullOfSubject(tm.Subject) - return m, nil - } - findMergeWarrant = func(_ context.Context, _ catalogueReader, _, _ string, number int, head string) (string, bool, string, error) { - for _, m := range tr.merges { - if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] { - return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil - } - } - return "", false, "the router holds no chosen merge ask for the pull request and head", nil - } - wasBeside := catchUpBeside - catchUpBeside = func(run func()) { run() } - t.Cleanup(func() { - trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside - settledMerges.Clear() - }) - return tr -} - -// aKeptMergeBefore is a merge the controller heard before the merges of a test: where its record begins, so what a trunk holds from -// then on can be a gap (novox/hq ADR 0463 §2). -func aKeptMergeBefore(t *testing.T, open *stores) { - t.Helper() - hear(t, open, catalogueMerge("mesh-catalog-seed", "app", t0.Add(-time.Hour)), t0.Add(-time.Hour)) -} - -func mergeKnown(t *testing.T, open *stores, commit string) bool { - t.Helper() - _, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit) - if err != nil { - t.Fatal(err) - } - return known -} - -// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said -// as caught up; a second start plans nothing new. -func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app") - hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second)) - hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second)) - if mergeKnown(t, open, "mesh-catalog-c02") { - t.Fatal("the merge to be caught up is known before the catch-up") - } - if (following{open: open}).catchUp(t.Context(), nil) { - t.Fatal("the catch-up says a trunk was left for later") - } - if !mergeKnown(t, open, "mesh-catalog-c02") { - t.Fatal("the merge the controller did not hear was not planned") - } - kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02") - if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") { - t.Fatalf("the merge does not say it was caught up: %q", said) - } - cutAt(t, open, t0.Add(10*time.Minute)) - ws, _ := walks(t, open) - if len(ws) == 0 { - t.Fatal("no walk answers the caught-up merge") - } - if _, in := ws[0].Modules["notes"]; !in { - t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules) - } - before := tr.events - if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before { - t.Fatalf("a second start read %d merge(s) in full again", tr.events-before) - } -} - -// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the -// earlier first, and the forge is read for that repository's trunk only. -func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app") - var order []string - inner := readMergeEvent - readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) { - order = append(order, tm.Commit) - return inner(ctx, owner, name, branch, tm) - } - t.Cleanup(func() { readMergeEvent = inner }) - tr.read = nil - heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)) - if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil { - t.Fatal(err) - } - for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} { - if !mergeKnown(t, open, c) { - t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c) - } - } - if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" { - t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order) - } - if len(tr.read) == 0 { - t.Fatal("the forge was not read for the trunk of the merge heard") - } - for _, r := range tr.read { - if r != "novox/mesh-catalog" { - t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read) - } - } -} - -// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window. -func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - modules := make([]string, 51) - for i := range modules { - modules[i] = "app" - } - aTrunkOf(t, "mesh-catalog", modules...) - // Fifty are known; the oldest is not. - for i := 2; i <= 51; i++ { - c := fmt.Sprintf("mesh-catalog-c%02d", i) - hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second)) - } - // Paged as the forge pages: fifty a page, and a trunk of another repository is empty (a short page, or none). - pages := trunkCommitsPage - trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) { - all, err := pages(ctx, owner, name, branch, 1) - if err != nil || page < 1 || (page-1)*50 >= len(all) { - return nil, err - } - return all[(page-1)*50 : min(page*50, len(all))], nil - } - t.Cleanup(func() { trunkCommitsPage = pages }) - (following{open: open}).catchUp(t.Context(), nil) - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("the oldest of fifty-one merges was read and planned, outside the window") - } -} - -// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and -// commit; a second start leaves that one condition open, observed twice; planning the gap clears it. -func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - aKeptMergeBefore(t, open) - keeper, _ := withConditionsInMemory(t) - tr := aTrunkOf(t, "mesh-catalog", "notes") - tr.approved["mesh-catalog-c01"] = false - catch := func() { (following{open: open}).catchUp(t.Context(), nil) } - catch() - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge with no Approve on record was planned") - } - raised := func() []conditions.Condition { - t.Helper() - all, err := keeper.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var out []conditions.Condition - for _, c := range all { - if c.Kind == kindMergeNotHeard { - out = append(out, c) - } - } - return out - } - one := raised() - if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") { - t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one) - } - catch() - if again := raised(); len(again) != 1 || again[0].Observations < 2 { - t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again) - } - tr.approved["mesh-catalog-c01"] = true - catch() - if !mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("the gap was not planned once its Approve was on record") - } - if left := raised(); len(left) != 0 { - t.Fatalf("planning the gap left its condition open: %+v", left) - } -} - -// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it. -func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - aKeptMergeBefore(t, open) - tr := aTrunkOf(t, "mesh-catalog", "notes") - tr.silent = true - if later := (following{open: open}).catchUp(t.Context(), nil); !later { - t.Fatal("a forge that did not answer counted as nothing to catch up") - } - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge was planned without the forge's word") - } - tr.silent = false - if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatalf("the next start did not catch up (left for later: %v)", later) - } -} - -// Finding 2 of the review: a gap that adds a module is a gap, judged as hearMerge judges it. With the operator's Approve on record it -// is heard (hearMerge asks the new module in its walk); without, it raises the condition and is not lost in silence. -func TestAGapThatAddsAModuleIsHeardOnItsWarrantAndRaisedWithout(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - aKeptMergeBefore(t, open) - keeper, _ := withConditionsInMemory(t) - tr := aTrunkOf(t, "mesh-catalog", "newmodule") - tr.approved["mesh-catalog-c01"] = false - catch := func() { (following{open: open}).catchUp(t.Context(), nil) } - catch() - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge adding a module was heard with no Approve on record") - } - all, err := keeper.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var raised int - for _, c := range all { - if c.Kind == kindMergeNotHeard { - raised++ - } - } - if raised != 1 { - t.Fatalf("a merge adding a module with no Approve raised %d condition(s), not one: it was lost in silence", raised) - } - tr.approved["mesh-catalog-c01"] = true - catch() - if !mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge adding a module was not heard once its Approve was on record") - } -} - -// A trunk with merges from before the controller began keeping merges, and merges since: only the later ones are gaps. -func TestAMergeFromBeforeTheRecordBeganIsHistoryAndNoCondition(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - keeper, _ := withConditionsInMemory(t) - tr := aTrunkOf(t, "mesh-catalog", "notes", "notes", "notes") - tr.approved["mesh-catalog-c01"], tr.approved["mesh-catalog-c02"], tr.approved["mesh-catalog-c03"] = false, false, false - // The record begins at the second merge: the first is two days older. - tr.merges[0].At = t0.Add(-48 * time.Hour) - hear(t, open, catalogueMerge("mesh-catalog-c02", "notes", t0.Add(time.Minute)), t0.Add(time.Minute)) - delete(tr.approved, "mesh-catalog-c02") - (following{open: open}).catchUp(t.Context(), nil) - // The first merge (two days before the record) was not even read from the forge: only the third, which is unknown, was. - if tr.events != 1 { - t.Fatalf("%d merge(s) were read from the forge: the one from before the record is history and is not read", tr.events) - } - all, err := keeper.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var raised int - for _, c := range all { - if c.Kind == kindMergeNotHeard { - raised++ - } - } - if raised != 1 { - t.Fatalf("%d condition(s) raised: only the merge since the record began (the third) is a gap", raised) - } -} - -// novox/hq ADR 0463 §2: the record begins at the earliest merge kept on ANY trunk. A trunk the controller kept no merge of has the -// merges made since as gaps (mesh-delivery #1 on novox), and the merges made before as history. -func TestATrunkWithNoKeptMergeHasItsLaterMergesAsGapsAndItsEarlierOnesAsHistory(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - // The record begins on another trunk, at t0 less an hour. - hear(t, open, repoMerge("one", "c-one", t0.Add(-time.Hour)), t0.Add(-time.Hour)) - tr := aTrunkOf(t, "mesh-catalog", "notes", "notes") - tr.merges[0].At = t0.Add(-72 * time.Hour) // before the record: history - (following{open: open}).catchUp(t.Context(), nil) - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge from before the record began was heard") - } - if !mergeKnown(t, open, "mesh-catalog-c02") { - t.Fatal("a merge since the record began, on a trunk with no merge kept, was not caught up") - } -} diff --git a/cmd/mesh-controller/catch_up_test.go b/cmd/mesh-controller/catch_up_test.go deleted file mode 100644 index eb7be1bb..00000000 --- a/cmd/mesh-controller/catch_up_test.go +++ /dev/null @@ -1,318 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// The pieces of the catch-up that need no store (novox/hq ADR 0457, issue 1336). - -func TestAMergeCommitsSubjectNamesItsPullRequest(t *testing.T) { - title, number, ok := pullOfSubject("Merge pull request 'Fix keycloak's sign-in (issue 1332)' (#378) from fix/1332-sign-in into main") - if !ok || number != 378 || title != "Fix keycloak's sign-in (issue 1332)" { - t.Fatalf("read %q %d %v", title, number, ok) - } - if b := headBranchOfSubject("Merge pull request 'x' (#378) from fix/1332-sign-in into main"); b != "fix/1332-sign-in" { - t.Fatalf("the head branch read as %q", b) - } - if _, _, ok := pullOfSubject("Merge branch 'main' of somewhere"); ok { - t.Fatal("a merge that names no pull request read as one") - } -} - -func TestOnlyTheLastFiftyMergesOfATrunkAreRead(t *testing.T) { - pages := map[int][]trunkCommit{} - var all []trunkCommit // newest first: 51 merges, a plain commit between each pair - for i := 51; i >= 1; i-- { - all = append(all, trunkCommit{Commit: fmt.Sprintf("merge-%02d", i), SecondParent: fmt.Sprintf("head-%02d", i), - Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from fix/x into main", i, i)}, - trunkCommit{Commit: fmt.Sprintf("plain-%02d", i)}) - } - for i := 0; i < len(all); i += 50 { - pages[i/50+1] = all[i:min(i+50, len(all))] - } - was := trunkCommitsPage - trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { return pages[page], nil } - t.Cleanup(func() { trunkCommitsPage = was }) - got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main") - if err != nil { - t.Fatal(err) - } - if len(got) != catchUpWindow || got[0].Commit != "merge-02" || got[len(got)-1].Commit != "merge-51" { - t.Fatalf("read %d merges from %s to %s: the oldest of 51 is outside the window", len(got), got[0].Commit, got[len(got)-1].Commit) - } - if got[0].Head != "head-02" { - t.Fatalf("a merge's head is its second parent, not %q", got[0].Head) - } -} - -func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) { - was := trunkCommitsPage - trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { - return nil, errors.New("the forge did not answer") - } - t.Cleanup(func() { trunkCommitsPage = was }) - if got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main"); err == nil { - t.Fatalf("a forge that did not answer read as %d merges: nothing to catch up", len(got)) - } -} - -func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) { - ok := approvedMerge(t) - if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil { - t.Errorf("the operator's Approve under the teller was refused: %v", err) - } - // 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken. - if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil { - t.Error("a warrant under mesh-delivery was accepted") - } - if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil { - t.Error("a warrant of an asker the controller takes none from was accepted") - } - declined := ok.warrant - declined.Option, declined.Label = "decline", "Decline" - if _, err := caughtUpWarrant(theDelivery, &ok.ask, &declined); err == nil { - t.Error("a Decline was accepted as a warrant") - } - unproved := ok.warrant - by := *unproved.By - by.Verified = "" - unproved.By = &by - if _, err := caughtUpWarrant(theDelivery, &ok.ask, &unproved); err == nil { - t.Error("an answer whose channel proves nobody was accepted") - } - if _, err := caughtUpWarrant(theDelivery, nil, nil); err == nil { - t.Error("a router that holds nothing was accepted") - } -} - -// aTellersAsk is the ask the delivery gate composes for a pull request, as the router holds it once the operator chose Approve -// (the gate's composeMergeAsk: its id from sdkdelivery.AskingID of the group and the heads, its Approve bound to mergeAct over the -// heads, the pull request and its head's first twelve digits in the details), with the warrant the router wrote. -func aTellersAsk(t *testing.T, group string, members ...sdkdelivery.Head) (asks.Ask, asks.Warrant) { - t.Helper() - key, err := sdkdelivery.MergeAskKey(group, members) - if err != nil { - t.Fatal(err) - } - id := sdkdelivery.AskingID(key, 1) - heads := map[string]string{} - var details []string - for _, m := range members { - owner, repo, _ := strings.Cut(m.Repository, "/") - heads[deliveryIDOf(owner, repo, m.Head)] = m.Head - details = append(details, fmt.Sprintf("%s#%d %q at %s: issues: none named; review: approved; reaches: no delivery plan yet", repo, m.Number, "title", m.Head[:12])) - } - bind := func(option string) string { - d, err := asks.ActDigest(mergeAct(id, option, heads)) - if err != nil { - t.Fatal(err) - } - return d - } - at := time.Date(2026, 10, 20, 12, 2, 1, 0, time.UTC) - ask := asks.Ask{ID: id, Headline: "Merge into main", Explanation: "passed its checks.", Who: asks.Operator, Expires: at.Add(24 * time.Hour), - OnExpiry: "nothing is merged; the ask is made again", Details: strings.Join(details, "\n"), - Options: []asks.Option{ - {ID: optionApprove, Label: "Approve", Does: "merges", Level: asks.Approve, Binds: bind(optionApprove)}, - {ID: "decline", Label: "Decline", Does: "nothing is merged", Level: asks.Approve, Binds: bind("decline")}}} - w := asks.Warrant{Ask: id, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve", - Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"}, - At: at.Add(time.Minute), AskDigest: ask.Digest()} - return ask, w -} - -// withRouter hands findMergeWarrant these asks as the router's records, by asker. -func withRouter(t *testing.T, records map[string]map[string]struct { - ask asks.Ask - w asks.Warrant -}) { - t.Helper() - wasIDs, wasAsk := routerAskIDs, routerAskOf - closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true }) - routerAskIDs = func(_ context.Context, _ catalogueReader, asker string) ([]string, error) { - var ids []string - for id := range records[asker] { - ids = append(ids, id) - } - return ids, nil - } - routerAskOf = func(_ context.Context, _ catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) { - r, ok := records[asker][id] - if !ok { - return nil, nil, nil - } - return &r.ask, &r.w, nil - } - t.Cleanup(func() { - routerAskIDs, routerAskOf = wasIDs, wasAsk - closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true }) - }) -} - -type aRouterRecord = struct { - ask asks.Ask - w asks.Warrant -} - -// The teller's asks, as the router held them on 2026-10-20: the id is the router's own key, rebuilt here from the group and the -// head the gate composed it for (mesh-controller#405 at 8a330afd, mesh-catalog#381 at 1c702422, mesh-host#124 at 52600ec7). It names -// neither the pull request nor the head, so the warrant is found by the heads the Approve is bound to. -func TestATellersWarrantIsFoundByTheHeadItsApproveIsBoundTo(t *testing.T) { - real := []struct { - group, key, owner, repo string - number int - head string - }{ - {"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "novox", "mesh-controller", 405, "8a330afd44ba852873339cf1210e90989dca46e1"}, - {"fix/1351-remove-todays-delivery-module", "merge-fix_1351_remove_todays_delivery_module-64a353-d93b97b9697b", "novox", "mesh-catalog", 381, "1c7024224dd77c5b603299378883e0993b2e2fdd"}, - {"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-6a59ba48da14-424e053ebc9c", "novox", "mesh-host", 124, "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"}, - } - teller := map[string]aRouterRecord{} - for _, r := range real { - ask, w := aTellersAsk(t, r.group, sdkdelivery.Head{Repository: r.owner + "/" + r.repo, Number: r.number, Head: r.head}) - if ask.ID != r.key { - t.Fatalf("the id of %s#%d is %q, and the router held it as %q: the fixture is not the router's", r.repo, r.number, ask.ID, r.key) - } - teller[ask.ID] = aRouterRecord{ask, w} - } - withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: teller}) - for _, r := range real { - why, found, reason, err := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, r.head) - if err != nil || !found || !strings.Contains(why, r.key) || !strings.Contains(why, "bound to the head of this pull request") { - t.Errorf("%s#%d: the warrant was not found: %q %v %q %v", r.repo, r.number, why, found, reason, err) - } - // Another head of the same pull request is another merge: the Approve was not given for it. - if _, found, _, _ := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, "ffffffffffffffffffffffffffffffffffffffff"); found { - t.Errorf("%s#%d: another head found the warrant of this one", r.repo, r.number) - } - } - // One pull request's Approve is none for another repository's merge at the same head digits. - if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-tools", 405, real[0].head); found { - t.Error("a merge of another repository found mesh-controller#405's warrant") - } -} - -// A group's ask binds several heads, and the router's record carries only twelve digits of the other members': the pull request is -// found in the ask as it was published, which the warrant's digest covers. -func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testing.T) { - a := sdkdelivery.Head{Repository: "novox/mesh-host", Number: 124, Head: "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"} - b := sdkdelivery.Head{Repository: "novox/mesh-controller", Number: 405, Head: "8a330afd44ba852873339cf1210e90989dca46e1"} - ask, w := aTellersAsk(t, "feat/r4b-sdk-users", a, b) - withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, w}}}) - for _, h := range []sdkdelivery.Head{a, b} { - owner, repo, _ := strings.Cut(h.Repository, "/") - why, found, reason, err := findMergeWarrant(t.Context(), nil, owner, repo, h.Number, h.Head) - if err != nil || !found || !strings.Contains(why, "a group's ask naming this pull request at this head") { - t.Errorf("%s#%d: not found in the group's ask: %q %v %q %v", repo, h.Number, why, found, reason, err) - } - } - if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 381, a.Head); found { - t.Error("a pull request the group's ask does not name found its warrant") - } - // A Decline is no warrant. - declined := w - declined.Option, declined.Label = "decline", "Decline" - withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, declined}}}) - if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-host", 124, a.Head); found { - t.Error("a Decline found a warrant") - } -} - -// A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do. -func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) { - was := trunkCommitsPage - t.Cleanup(func() { trunkCommitsPage = was }) - trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { - if page > 1 { - return nil, nil - } - return []trunkCommit{{Commit: "m2", SecondParent: "h2", Subject: "Merge pull request 'two' (#2) from fix/b into main"}, - {Commit: "p"}, {Commit: "m1", SecondParent: "h1", Subject: "Merge pull request 'one' (#1) from fix/a into main"}}, nil - } - got, err := readTrunkMerges(t.Context(), "novox", "x", "main") - if err != nil || len(got) != 2 || got[0].Commit != "m1" || got[1].Commit != "m2" { - t.Fatalf("a short trunk read as %+v %v", got, err) - } - trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { return nil, nil } - if got, err := readTrunkMerges(t.Context(), "novox", "x", "main"); err != nil || len(got) != 0 { - t.Fatalf("an empty trunk read as %+v %v", got, err) - } -} - -// ADR 0253: what the operator reads of a merge not heard is plain: no hash, no identifier, nothing to do "by hand". -func TestTheConditionOfAMergeNotHeardIsInPlainWords(t *testing.T) { - keeper, _ := withConditionsInMemory(t) - tm := trunkMerge{Commit: "c0880c4d1f2e3a4b5c6d7e8f9a0b1c2d3e4f5a6b", Head: "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372"} - (following{}).raiseMergeNotHeard(t.Context(), "novox/mesh-catalog", "main", tm, link.SourceMoved{Number: 378}, - "the router holds no chosen merge ask of mesh-delivery-teller or mesh-delivery that covers the head b9104454f9a5") - open, err := keeper.Open(t.Context()) - if err != nil || len(open) != 1 { - t.Fatalf("opened %d conditions, %v", len(open), err) - } - c := open[0] - if why, ok := conditions.PlainWords(conditions.Words{Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs, - Resolved: c.Resolved}); !ok { - t.Fatalf("the condition's words are not plain: %s\n%q\n%q\n%q", why, c.Headline, c.Explanation, c.Needs) - } -} - -// The trunk's history lists the merges made inside a pull request's branch too. On novox (2026-10-20) 15 of the 15 non-pull-request -// "gaps" of mesh-catalog and mesh-controller were such merges; only a pull request's merge into this trunk is a merge to catch up. -func TestAMergeMadeInsideABranchIsNoMergeOfTheTrunk(t *testing.T) { - was := trunkCommitsPage - t.Cleanup(func() { trunkCommitsPage = was }) - trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { - if page > 1 { - return nil, nil - } - return []trunkCommit{ - {Commit: "c0880c4d", SecondParent: "b9104454", Subject: "Merge pull request 'Read the grant file' (#378) from fix/1332 into main"}, - {Commit: "6394a62a", SecondParent: "x1", Subject: "Merge remote-tracking branch 'origin/main' into fix/1081-withdraw"}, - {Commit: "896f1d28", SecondParent: "x2", Subject: "Merge feat/agents-write-git-ssh into feat/agents-write-git-ssh-f (issue 955 F)"}, - {Commit: "4d03755c", SecondParent: "x3", Subject: "Bring the branch up to main after #354 (issue 1047)"}, - {Commit: "other", SecondParent: "x4", Subject: "Merge pull request 'Elsewhere' (#5) from fix/y into nox-mesh"}, - }, nil - } - got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main") - if err != nil || len(got) != 1 || got[0].Commit != "c0880c4d" { - t.Fatalf("read %+v %v: only keycloak's pull request merge into main is a merge to catch up", got, err) - } -} - -// novox/hq ADR 0463 §2: the record of merges begins at the earliest merge the controller holds on any trunk. A merge before it is -// history on every trunk; a trunk with no merge kept of its own still has the merges made since as gaps (mesh-delivery #1 on novox). -func TestOnlyTheMergesSinceTheRecordBeganAreGapsOnEveryTrunk(t *testing.T) { - first := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) // the first merge it heard, on some trunk of the mesh - trunk := func(prefix string) []trunkMerge { - return []trunkMerge{ - {Commit: prefix + "-old-1", At: first.Add(-30 * 24 * time.Hour)}, - {Commit: prefix + "-old-2", At: first.Add(-2 * time.Hour)}, - {Commit: prefix + "-within-grace", At: first.Add(-mergeGrace / 2)}, - {Commit: prefix + "-since", At: first.Add(24 * time.Hour)}, - } - } - // Two trunks, one of which the controller has kept no merge of: both read by the one starting point. - for _, prefix := range []string{"kept-trunk", "trunk-with-no-kept-merge"} { - var commits []string - for _, m := range sinceTheRecord(trunk(prefix), true, first) { - commits = append(commits, m.Commit) - } - if want := prefix + "-within-grace " + prefix + "-since"; strings.Join(commits, " ") != want { - t.Errorf("%s: the gaps are %v, want %q: earlier ones are history on every trunk, later ones are gaps", prefix, commits, want) - } - } - if none := sinceTheRecord(trunk("x"), false, time.Time{}); len(none) != 0 { - t.Errorf("with no merge kept anywhere there are %d gap(s), but no record to have one in", len(none)) - } -} diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index fa28d8ae..5c58ff7a 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -54,7 +54,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. // The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build // (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes. // And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds). - go copySentBuildsSoon(ctx, inv) return nil } @@ -62,39 +61,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. // succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run. const kindSentBuildsNotCopied = "sent-builds-not-copied" -// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon -// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails, -// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key -// that names a commit and empties each column it has copied. -func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) { - failed := false - for { - _, err := inv.CopySentBuilds(ctx) - o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied, - Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, - Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "} - if err != nil { - fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err) - if keeper := conditionsFrom; keeper != nil && !failed { - o.Summary += err.Error() - if _, oerr := keeper.Observe(ctx, o); oerr == nil { - failed = true - } - } - } else { - if keeper := conditionsFrom; keeper != nil && failed { - _, _ = keeper.Clear(ctx, o.Key(), "copied") - } - return - } - select { - case <-ctx.Done(): - return - case <-time.After(chainMoveRetry): - } - } -} - // chainMoveRetry is how long the copy waits before it tries again. const chainMoveRetry = 30 * time.Second diff --git a/cmd/mesh-controller/chain_integration_test.go b/cmd/mesh-controller/chain_integration_test.go deleted file mode 100644 index 57a72b45..00000000 --- a/cmd/mesh-controller/chain_integration_test.go +++ /dev/null @@ -1,232 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "fmt" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there, -// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live -// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's -// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send -// is recorded as the controller records one it made, and the gate's facts are the store's. -func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if _, err := assign(ctx, open, "anchor", "notes"); err != nil { - t.Fatal(err) - } - - var sent [][]string - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - digest := fmt.Sprintf("d-%s-%d", node, len(sent)) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, - map[string]string{"notes": current["notes"].Commit}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - // A merge of notes is heard, and its window closes: the batch is cut into the walk. - hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second)) - cutAt(t, open, t0.Add(time.Second+90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walk(s) after the cut", len(ws)) - } - rollout := ws[0].ID - - // The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the - // outcome in as it does. - asking, err := inv.PlanByID(ctx, rollout) - if err != nil { - t.Fatal(err) - } - for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ { - advancePlans(ctx, open) - if asking, err = inv.PlanByID(ctx, rollout); err != nil { - t.Fatal(err) - } - } - buildID := asking.Modules["notes"].Build - if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) { - t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID) - } - - // The walk sends the tier to its first machine. - for try := 0; try < 3 && len(sent) == 0; try++ { - advancePlans(ctx, open) - } - if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) { - t.Fatalf("sent %v", sent) - } - - // One read of the assignment. - on, err := inv.AssignmentsOn(ctx, "anchor") - a, ok := assignmentOf(on, "notes") - if err != nil || !ok { - t.Fatalf("the assignments of anchor: %+v %v", on, err) - } - // Beside it a module pulled in by the assignment, which no send of this walk put anywhere. - if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") { - t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard) - } - if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout || - a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() { - t.Fatalf("the assignment names %+v, not commit c-notes, build %s, rollout %s", a, buildID, rollout) - } - // One read of the rollout. - p, err := inv.PlanByID(ctx, rollout) - if err != nil { - t.Fatal(err) - } - wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}} - if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 || - p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil { - t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange) - } - // And one read of the build names the same commit. - built, found, err := inv.BuildByID(ctx, buildID) - if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" { - t.Fatalf("the build: %+v %v %v", built, found, err) - } - // The verbs say the links. - if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") || - !strings.Contains(lines[1], buildID) { - t.Fatalf("`plans` says %v", lines) - } - if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) { - t.Fatalf("`node show` says %v", lines) - } -} - -// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks -// nothing (issue 1095). -func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - register(t, open, catalogue.Manifest{Module: "notes", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "notes"); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil { - t.Fatal(err) - } - node := nodeID(t, open, "anchor") - inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}}) - if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil { - t.Fatal(err) - } - // Sent again by hand, with no rollout: the assignment still names the rollout that put the build there. - if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil { - t.Fatal(err) - } - on, _ := inv.AssignmentsOn(ctx, "anchor") - if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" { - t.Fatalf("the assignment names %+v", on) - } - // A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit. - if err := inv.RecordSent(ctx, node, "d3", nil); err != nil { - t.Fatal(err) - } - if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() { - t.Fatalf("a send of unknown builds left %+v", on) - } - // A module sent a commit no build is kept of is named by its commit alone. - if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil { - t.Fatal(err) - } - if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" || - runOf(on, "notes").Rollout != "" { - t.Fatalf("the assignment after a push of c2 by hand names %+v", on) - } -} - -// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it. -func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) { - for _, a := range on { - if a.Module == module { - return a, true - } - } - return chain.Assignment{}, false -} - -func runOf(on []chain.Assignment, module string) chain.Assignment { - a, _ := assignmentOf(on, module) - return a -} - -// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered -// through the `assign` verb for a module whose build is in flight, one read of `.` says state pending with the -// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied, -// and the table of pending assignments is read by nobody. No key is written by hand. -func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatalf("an assignment while its build runs was refused: %v", err) - } - waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") - if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() || - waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" || - !strings.HasPrefix(waiting.Pending.ID, "p-") { - t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err) - } - if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") { - t.Fatalf("a module only waiting is assigned: %v", got) - } - - if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") - if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil || - made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID { - t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err) - } - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("the build registered sensors and it was not assigned") - } -} diff --git a/cmd/mesh-controller/chain_links.go b/cmd/mesh-controller/chain_links.go deleted file mode 100644 index b6d01adb..00000000 --- a/cmd/mesh-controller/chain_links.go +++ /dev/null @@ -1,121 +0,0 @@ -package main - -import ( - "context" - "fmt" - "slices" - "time" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a -// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of -// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the -// build before and after. - -// rolloutModules is what a plan sends of each module: the commit and the build of it. -func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule { - out := make(map[string]inventory.RolloutModule, len(p.Modules)) - for m, s := range p.Modules { - if s != nil && s.Commit != "" { - out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build} - } - } - return out -} - -// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout. -func sendingAs(ctx context.Context, p *inventory.Plan) context.Context { - return inventory.WithRollout(ctx, p.ID, rolloutModules(p)) -} - -// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit, -// else the commit the machine was last sent for it. Read before the send, which writes the new ones. -func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref { - was := map[string]chain.Ref{} - if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known { - for m, commit := range sent { - was[m] = chain.Ref{Commit: commit} - } - } - if ran, err := inv.AssignmentRuns(ctx, node); err == nil { - for m, r := range ran { - was[m] = r - } - } - return was -} - -// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and -// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the -// module is replaced, so a send made again says it once. -func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) { - for _, m := range modules { - s := p.Modules[m] - if s == nil { - continue - } - change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}} - if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 { - change.Before = p.Changed[at].Before - p.Changed[at] = change - } else { - p.Changed = append(p.Changed, change) - } - if s.Build != "" && !slices.Contains(p.Builds, s.Build) { - p.Builds = append(p.Builds, s.Build) - } - } - if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) { - sent := at.UTC() - p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent}) - } -} - -// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none. -func refSaid(r chain.Ref) string { - switch { - case r.Build == "" && r.Commit == "": - return "nothing" - case r.Build == "": - return "commit " + short(r.Commit) - } - return r.Build + " (" + short(r.Commit) + ")" -} - -// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after. -func changedLines(p inventory.Plan) []string { - if len(p.Changed) == 0 { - return nil - } - out := []string{" changed:"} - for _, c := range p.Changed { - out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After))) - } - return out -} - -// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the -// rollout that put it there and since when. -func runLines(assigned []chain.Assignment) []string { - var out []string - for _, a := range assigned { - if a.Build == "" && a.Commit == "" { - continue - } - line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit})) - if a.Rollout != "" { - line += ", sent by " + a.Rollout - } - if !a.Since.IsZero() { - line += " at " + a.Since.Local().Format("2006-01-02 15:04") - } - out = append(out, line) - } - if len(out) == 0 { - return nil - } - return append([]string{" what they run, as the chain names it:"}, out...) -} diff --git a/cmd/mesh-controller/chain_links_test.go b/cmd/mesh-controller/chain_links_test.go deleted file mode 100644 index 8652bd60..00000000 --- a/cmd/mesh-controller/chain_links_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one -// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the -// rollout carried once each, and one step for a machine and a tier. -func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) { - at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{ - "notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}} - was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}} - - noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at) - noteChanges(&p, "laptop", []string{"notes"}, nil, at) - // The send noted again, after what was sent had become what the machine runs: the first before stands. - noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute)) - - want := []chain.Change{ - {Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}}, - {Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}}, - {Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}}, - } - if len(p.Changed) != len(want) { - t.Fatalf("changed %+v", p.Changed) - } - for k := range want { - if p.Changed[k] != want[k] { - t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k]) - } - } - if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" { - t.Errorf("builds %v", p.Builds) - } - if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" { - t.Errorf("steps %+v", p.Steps) - } - lines := changedLines(p) - if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") || - !strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") { - t.Errorf("changed lines %q", lines) - } - if got := changedLines(inventory.Plan{}); got != nil { - t.Errorf("a plan that changed nothing says %q", got) - } -} - -func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) { - at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - lines := runLines([]chain.Assignment{ - {Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at}, - {Node: "anchor", Module: "web", Commit: "c3"}, - {Node: "anchor", Module: "never-sent"}, - }) - if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") || - !strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") { - t.Fatalf("lines %q", lines) - } - if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil { - t.Fatal("a module never sent is said to run something") - } -} diff --git a/cmd/mesh-controller/chain_no_table_integration_test.go b/cmd/mesh-controller/chain_no_table_integration_test.go deleted file mode 100644 index aab6ca2d..00000000 --- a/cmd/mesh-controller/chain_no_table_integration_test.go +++ /dev/null @@ -1,77 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" -) - -// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR -// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's -// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the -// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays. -func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if _, err := assign(ctx, open, "anchor", "notes"); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second)) - cutAt(t, open, t0.Add(91*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walk(s) after the cut", len(ws)) - } - rollout := ws[0].ID - asking, err := inv.PlanByID(ctx, rollout) - if err != nil { - t.Fatal(err) - } - for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ { - advancePlans(ctx, open) - if asking, err = inv.PlanByID(ctx, rollout); err != nil { - t.Fatal(err) - } - } - buildID := asking.Modules["notes"].Build - if buildID == "" || len(*asked) != 1 { - t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID) - } - - // What the live path made is in the chain. - if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" { - t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err) - } - if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout { - t.Fatalf("the rollout is not read from the chain: %+v %v", p, err) - } - if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil { - t.Fatal(err) - } else if _, ok := assignmentOf(on, "notes"); !ok { - t.Fatalf("the assignment is not read from the chain: %+v", on) - } - - // And the catalog of the store holds none of them. - have := map[string]bool{} - for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { - have[strings.Trim(row[0], `"`)] = true - } - for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} { - if have[table] { - t.Errorf("the table %s is in the controller's store", table) - } - } - for _, table := range []string{"build_gate", "assignment_generation"} { - if !have[table] { - t.Errorf("the table %s, which stays, is gone", table) - } - } - if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { - t.Errorf("the function raise_assignment_generation is in the store: %v", rows) - } -} diff --git a/cmd/mesh-controller/changeplan.go b/cmd/mesh-controller/changeplan.go deleted file mode 100644 index da60adf4..00000000 --- a/cmd/mesh-controller/changeplan.go +++ /dev/null @@ -1,192 +0,0 @@ -package main - -import ( - "fmt" - "path" - "slices" - "sort" - "strings" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed -// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by -// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module -// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary -// send. A pull request's check posts it with its verdict; the release a merge makes follows the same -// planner, so the two can be compared. - -// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read. -type policyOf func(module string) (inventory.Upgrade, bool) - -// changePlanOf is the change plan of a reach: pure, so it is tested without a store. -func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan { - p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(), - New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers} - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - // The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490). - holder := "" - for _, e := range entries { - if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 { - holder = e.On[0] - } - } - var granting []string - machines := map[string]*link.MachinePlan{} - machine := func(name string) *link.MachinePlan { - if machines[name] == nil { - machines[name] = &link.MachinePlan{Machine: name} - } - return machines[name] - } - for _, tier := range r.Plan.Tiers { - for _, name := range tier { - e, held := byName[name] - if !held { - continue - } - u, known := policy(name) - waits := known && !u.RollOut - for _, on := range e.On { - if waits { - machine(on).Waits = append(machine(on).Waits, name) - } else { - machine(on).Receives = append(machine(on).Receives, name) - } - } - if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 && - !slices.Contains(granting, name) { - granting = append(granting, name) - } - switch { - case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0: - // Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine - // until a person takes the step, so merging it is a promise to take it. - p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+ - "nothing else is sent to %s either — unless the new build turns out the same as the one running there "+ - "(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", "))) - case name == "nats" || catalogue.ProvidesBus(e.Manifest): - p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send") - case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData): - // Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377). - p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+ - "its move recreates none of its containers and moves none of its data, which the walk judges and then sends", - name, orNone(u.From))) - case waits && len(e.On) > 0: - p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name, - orNone(u.From))) - } - if len(e.Manifest.Provides) > 0 && len(e.On) > 0 { - var offers []string - for _, o := range e.Manifest.Provides { - offers = append(offers, o.Name) - } - p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it", - name, strings.Join(offers, ", "))) - } - if e.Manifest.Data != nil && len(e.On) > 0 { - p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name)) - } - } - } - if len(granting) > 0 { - // Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's - // user list. Whether it does is known only once the builds are registered, so it is said as a rule. - p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+ - "subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+ - "machine", grantsStepWord, holder, strings.Join(granting, ", "))) - } - var names []string - for name := range machines { - names = append(names, name) - } - sort.Strings(names) - for _, name := range names { - p.Machines = append(p.Machines, *machines[name]) - } - p.Summary = summaryOf(p) - p.Class = changeClassOf(p, entries) - return p -} - -// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2). -const classNone = "none" - -// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves, -// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf -// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the -// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds -// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a -// resolver claim its manifest makes. -func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string { - if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 { - return classNone - } - modules := append(append([]string{}, p.Moved...), p.Dependents...) - for _, dir := range p.New { - if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" { - modules = append(modules, name) - } - } - return classOfModules(modules, entries) -} - -// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine -// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself. -const grantsStepWord = "grants step" - -// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step. -const busUpgradeNeeded = "merging this needs a person's bus upgrade" - -// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves. -func summaryOf(p link.ChangePlan) string { - if len(p.Moved) == 0 && len(p.New) == 0 { - return "builds nothing: the change touches no module of the mesh's graph" - } - var parts []string - what := strings.Join(p.Moved, ", ") - if len(p.Dependents) > 0 { - what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents)) - } - if len(p.New) > 0 { - if what != "" { - what += ", " - } - what += "new: " + strings.Join(p.New, ", ") - } - var to []string - for _, m := range p.Machines { - if len(m.Receives) > 0 { - to = append(to, m.Machine) - } - } - if len(to) > 0 { - parts = append(parts, "builds "+what+" → "+strings.Join(to, ", ")) - } else { - parts = append(parts, "builds "+what+", sent nowhere") - } - bus := "no bus step" - for _, s := range p.Steps { - switch { - case strings.HasPrefix(s, busUpgradeNeeded): - bus = busUpgradeNeeded - case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step": - bus = "a bus step" - } - } - parts = append(parts, bus) - waiting := 0 - for _, m := range p.Machines { - waiting += len(m.Waits) - } - if waiting > 0 { - parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting)) - } - return strings.Join(parts, "; ") -} diff --git a/cmd/mesh-controller/changeplan_test.go b/cmd/mesh-controller/changeplan_test.go deleted file mode 100644 index bb250865..00000000 --- a/cmd/mesh-controller/changeplan_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in -// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a -// person, a provider whose consumers follow it. -func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) { - const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git" - entry := func(name, path string, on ...string) inventory.Entry { - e := fromRepo(name, catalogue_, path) - e.Source.BuiltFrom = "old" - e.On = on - return e - } - nats := entry("nats", "modules/nats", "anchor") - nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - gitea := entry("gitea", "modules/gitea", "anchor") - held := entry("photos", "modules/photos", "anchor", "laptop") - tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools") - tools.On = []string{"anchor", "laptop"} - entries := []inventory.Entry{nats, gitea, held, tools} - edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}} - policy := func(module string) (inventory.Upgrade, bool) { - if module == "photos" { - return inventory.Upgrade{RollOut: false, From: "a person"}, true - } - return inventory.Upgrade{RollOut: true}, true - } - plan := func(paths ...string) link.ChangePlan { - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths} - return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy) - } - - p := plan("modules/gitea/index.ts") - if p.Summary != "builds gitea → anchor; no bus step" { - t.Errorf("one module's change reads %q", p.Summary) - } - - p = plan("modules/nats/Dockerfile", "modules/photos/x.js") - if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") || - !strings.Contains(p.Summary, "(+1 dependent(s))") { - t.Errorf("the bus and a held module read %q", p.Summary) - } - got := map[string]string{} - for _, m := range p.Machines { - got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",") - } - // The bus first, what stands on it after: the build plan's order, per machine. - if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" { - t.Errorf("the deploy plan reads %v", got) - } - text := strings.Join(p.Steps, "\n") - // Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a - // person runs the bus's step, and the verb that does. - for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor", - "mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person", - "nats provides mesh-bus"} { - if !strings.Contains(text, want) { - t.Errorf("the steps do not say %q:\n%s", want, text) - } - } - - p = plan("merge-check.sh") - if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" { - t.Errorf("a root file's plan reads %+v", p) - } -} diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index f3d84c34..8b1d093e 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -38,10 +38,8 @@ type checkScope struct { New []string // Manifests are the moved modules' and the new ones' manifests in the change's tree. Manifests []string - // Width is how many modules a merge would build, in how many tiers; Unread the changed files no - // module's build reads. - Width, Tiers int - Unread []string + // Unread the changed files no module's build reads. + Unread []string // Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the // core's, or the change adds a module to it. Mesh bool @@ -62,8 +60,7 @@ func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][] Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid} r := reachOfMerge(m, entries, read, edges) - s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, - Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r} + s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, Reach: r} for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) { s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile)) } diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index 96d134cb..f35ea93f 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -93,60 +93,3 @@ func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) { t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests) } } - -// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request -// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no -// module in both, and what stands on a touched module is named as its dependents in both. -func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) { - const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git" - nats := fromRepo("nats", catalogue, "modules/nats") - nats.Source.BuiltFrom = "old" - gitea := fromRepo("gitea", catalogue, "modules/gitea") - gitea.Source.BuiltFrom = "old" - tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools") - entries := []inventory.Entry{nats, gitea, tools} - // node-tools stands on the bus's image; a code edge, so a plan takes it along. - edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}} - read := map[string][]inventory.ReadRepository{} - - for _, c := range []struct { - what string - paths []string - moved, dependents string - width int - unread string - }{ - {"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"}, - {"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""}, - {"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"}, - } { - p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths} - s := pullScope(p, entries, read, edges) - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths} - r := reachOfMerge(m, entries, read, edges) - if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") { - t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved) - } - if got := strings.Join(s.Dependents, ","); got != c.dependents { - t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents) - } - if s.Width != c.width || s.Width != len(r.Plan.Modules) { - t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width) - } - if got := strings.Join(s.Unread, ","); got != c.unread { - t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread) - } - } - - // A repository whose build another module's recipe packages: that module is reached through the - // planner's `read`, and what stands on it after it. - read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}} - s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head", - Paths: []string{"src/index.ts"}}, entries, read, edges) - if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 { - t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests) - } -} - -// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the -// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error. diff --git a/cmd/mesh-controller/compose_input.go b/cmd/mesh-controller/compose_input.go index 7e176e7b..e405918e 100644 --- a/cmd/mesh-controller/compose_input.go +++ b/cmd/mesh-controller/compose_input.go @@ -245,41 +245,8 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento if out.Sequence, err = inv.Sequence(ctx, n.ID); err != nil { return out, err } - if out.Epoch, err = inv.SentEpoch(ctx, n.ID); err != nil { - return out, err - } - if out.Generation, err = inv.SentGeneration(ctx, n.ID); err != nil { - return out, err - } - if out.ReadsModules, err = inv.ReadsModules(ctx, n.ID); err != nil { - return out, err - } - if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil { - return out, err - } - runs, err := inv.AssignmentRuns(ctx, n.Name) - if err != nil { - return out, err - } - if len(runs) > 0 { - out.Runs = make(map[string]compose.Run, len(runs)) - for module, ref := range runs { - out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build} - } - } - // A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its - // manifest and the commit, applied to this node's own resolution only. - // What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it). - kept, err := recordedKept(ctx, open, n.Name) - if err != nil { - *refused = err - } else if len(kept) > 0 { - out.KeptBuilds = kept - if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil { - *refused = err - out.KeptBuilds, out.Kept = nil, nil - } - } + // Every node-engine of release 6 reads a module per resource and a build per module. + out.ReadsModules = true stated, has, err := inv.HealthOf(ctx, n.Name) if err != nil { return out, err @@ -291,18 +258,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento return out, nil } -// keptManifests is the manifest of each module kept at the build a machine runs: keepRecorded's reads, and its refusals in its -// words, without putting any of it into a shelf (that is the node's own Kept, applied by the composition). -func keptManifests(ctx context.Context, open *stores, node string, kept map[string]string) (map[string]catalogue.Manifest, error) { - // applyKept puts each manifest into the shelf it is given and refuses a build that cannot be read; the shelf here is a - // scratch one, whose entries are the kept manifests. - scratch := map[string]catalogue.Manifest{} - if err := applyKept(ctx, open, node, kept, scratch); err != nil { - return nil, err - } - return scratch, nil -} - func readTunnels(ctx context.Context, inv *inventory.Inventory) (map[string]compose.FoundTunnel, error) { found, err := inv.Tunnels(ctx) if err != nil { diff --git a/cmd/mesh-controller/compose_input_integration_test.go b/cmd/mesh-controller/compose_input_integration_test.go index 6e4f190c..d26ac56b 100644 --- a/cmd/mesh-controller/compose_input_integration_test.go +++ b/cmd/mesh-controller/compose_input_integration_test.go @@ -6,11 +6,9 @@ import ( "reflect" "strings" "testing" - "time" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" - "github.com/novox/mesh-controller/internal/link" ) // 1382: the adapter's Input names, for every node, what the inventory answers for it, each compared with the read named @@ -132,69 +130,6 @@ func nodePorts(in compose.Input, name string) map[string]map[int]int { return n.Ports } -// 1382/AC2: a node keeping a recorded module at the build it runs has that module in Kept and KeptBuilds, and the Input's -// Catalogue is the current shelf, not the kept manifest (R4a review round 1, finding 1). -func TestANodeKeepingARecordedBuildHasItInKeptAndTheCatalogueStaysCurrent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, map[string][2]string{"server": {image, ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - if _, err := inv.Assign(ctx, "anchor", "postgres"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"postgres": "c1111111"}); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), - map[string][2]string{"server": {image, "runtime"}})); err != nil { - t.Fatal(err) - } - - // A send that is not a person's push keeps the build the machine runs. - in, err := readComposeInput(keepingRecorded(ctx), open) - if err != nil { - t.Fatal(err) - } - anchor, _ := nodeOf(in.Input, "anchor") - if anchor.KeptBuilds["postgres"] != "c1111111" { - t.Fatalf("anchor keeps %v, want postgres at c1111111", anchor.KeptBuilds) - } - kept, found := anchor.Kept["postgres"] - if !found { - t.Fatalf("the kept manifest of postgres is not in anchor's Kept: %v", anchor.Kept) - } - current, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(in.Catalogue["postgres"], current["postgres"]) { - t.Error("the Input's Catalogue is not the current shelf") - } - if reflect.DeepEqual(kept, current["postgres"]) { - t.Error("the kept manifest is the current one: nothing is kept") - } - if laptop, _ := nodeOf(in.Input, "laptop"); len(laptop.Kept) != 0 || len(laptop.KeptBuilds) != 0 { - t.Errorf("laptop keeps what only anchor runs: %v", laptop.KeptBuilds) - } - - // A person's push keeps nothing. - pushed, err := readComposeInput(ctx, open) - if err != nil { - t.Fatal(err) - } - if a, _ := nodeOf(pushed.Input, "anchor"); len(a.Kept) != 0 || len(a.KeptBuilds) != 0 { - t.Errorf("a person's push keeps %v", a.KeptBuilds) - } -} - // 1382/AC3: the seats on record, the tunnels, the adopted tunnel, its carried peers and the nodes joining equal the // inventory's answers; the range and the suffix are read from the environment once, here. func TestTheAdaptersInputCarriesTheNetworkAsTheInventoryHoldsIt(t *testing.T) { diff --git a/cmd/mesh-controller/compose_sdk.go b/cmd/mesh-controller/compose_sdk.go index 0a8f6b5e..408ae511 100644 --- a/cmd/mesh-controller/compose_sdk.go +++ b/cmd/mesh-controller/compose_sdk.go @@ -39,24 +39,10 @@ func withInputCache(ctx context.Context) context.Context { return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}}) } -// freshInput is ctx for one composition whose Input is read afresh and kept only until the composition makes something: a -// send that is composed after another one made and recorded things (a push's machines, one after the other). -func freshInput(ctx context.Context) context.Context { - return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}}) -} - // keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs // (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send. func keptSignature(ctx context.Context) string { - skip, on := keptExcept(ctx) - var names []string - for m := range skip { - names = append(names, m) - } - sort.Strings(names) - every, _ := ctx.Value(keepEveryBuildKey{}).(string) - settings, _ := ctx.Value(keepForSettingsKey{}).(string) - return fmt.Sprintf("kept=%v;except=%v;every=%s;settings=%s", on, names, every, settings) + return "" } // inputFor is the Input for ctx's operation. diff --git a/cmd/mesh-controller/compose_sdk_test.go b/cmd/mesh-controller/compose_sdk_test.go index 8d6a3cb8..f20caa22 100644 --- a/cmd/mesh-controller/compose_sdk_test.go +++ b/cmd/mesh-controller/compose_sdk_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "reflect" "strings" "testing" @@ -86,26 +85,3 @@ func TestADeclarationThatStandsInForAValueIsStillForeseen(t *testing.T) { } } } - -// 1386: an Input read for a person's push is never the one read for a send that keeps recorded builds, and the cache is -// emptied by what a send made. -func TestAnInputIsCachedPerWhatItKeepsAndForgottenAfterASend(t *testing.T) { - ctx := withInputCache(context.Background()) - plain := keptSignature(ctx) - kept := keptSignature(keepingRecorded(ctx)) - bus := keptSignature(keepingRecorded(ctx, "nats")) - every := keptSignature(keepingEveryBuild(ctx, "anchor")) - for name, sig := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} { - for other, o := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} { - if name != other && sig == o { - t.Errorf("%s and %s read the same Input", name, other) - } - } - } - cache := ctx.Value(inputCacheKey{}).(*inputCache) - cache.by[plain] = &meshInput{} - forgetInput(ctx) - if len(cache.by) != 0 { - t.Fatal("what a send made left the Input it read in the cache") - } -} diff --git a/cmd/mesh-controller/composed_integration_test.go b/cmd/mesh-controller/composed_integration_test.go new file mode 100644 index 00000000..b9d88388 --- /dev/null +++ b/cmd/mesh-controller/composed_integration_test.go @@ -0,0 +1,143 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "sync" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +// A mesh that composes (novox/hq ADR 0539): the real store (theCatalogue: sudo from the catalogue, mesh-host from the +// node-engine's repository, the private network on anchor and laptop), the controller's signing key in its own +// identity store, and a real bus server this process composes onto, its lease taken by the act as a command's is. Every +// subject published on the bus is heard, so a test can say nothing went to a node's declaration subject. +type composingMesh struct { + open *stores + js jetstream.JetStream + conn *nats.Conn + mu sync.Mutex + heard []string +} + +func aComposingMesh(t *testing.T) *composingMesh { + t.Helper() + open := theCatalogue(t) + ident := identity.ForTest(t) + if _, err := ident.Establish(t.Context()); err != nil { + t.Fatal(err) + } + s := testbus.Start(t) + t.Setenv(broker.NATSVar, s.ClientURL()) + was := theLease + theLease = &actor{} + t.Cleanup(func() { theLease.release(); theLease = was }) + conn, err := nats.Connect(s.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + m := &composingMesh{open: open, js: js, conn: conn} + sub, err := conn.Subscribe(">", func(msg *nats.Msg) { + m.mu.Lock() + m.heard = append(m.heard, msg.Subject) + m.mu.Unlock() + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + legacy, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + // The controller's seat events, kept as the mesh's events stream keeps them. + if _, err := legacy.AddStream(&nats.StreamConfig{Name: "CONTROLLER_EVENTS", Subjects: []string{"mesh.seat.mesh-controller.event.>"}}); err != nil { + t.Fatal(err) + } + wasEvents := checkEvents + checkEvents = link.OverNATS{Conn: conn, JS: legacy} + t.Cleanup(func() { checkEvents = wasEvents }) + withTrunkList(t, true) + return m +} + +// trunkKeys are a done key and the indexer's trunk value naming it. +type trunkKeys struct { + keyBuckets + trunk sdkdelivery.Module +} + +func (k trunkKeys) ModuleAt(context.Context, string, string) (sdkdelivery.Module, error) { + return k.trunk, nil +} + +// 1945/AC14: the declarations' stream stays while one node's have names another node-engine than the one registered, +// and goes, said once, when every node's names it (ADR 0539 clause 11). +func TestTheDeclarationsStreamGoesOnceEveryNodeRunsTheNewEngine(t *testing.T) { + m := aComposingMesh(t) + ctx := t.Context() + legacy, err := m.conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := legacy.AddStream(&nats.StreamConfig{Name: broker.DeclarationsStream, Subjects: []string{"mesh.node.*.declare"}, + MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + // The new node-engine registered at its fingerprint, from its done key. + n6 := fpOf('6') + keys, trunk := aKeyFor("mesh-host", "6", n6, "novox/mesh-host", "", "f00d0123456789ab") + if _, err := liveDoneKeySteps(m.open, trunkKeys{keyBuckets: keys, trunk: trunk}).take(ctx, "mesh-host"); err != nil { + t.Fatal(err) + } + if _, err := m.js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: sdkdelivery.HaveBucket, History: sdkdelivery.HaveHistory}); err != nil { + t.Fatal(err) + } + haves, err := m.js.KeyValue(ctx, sdkdelivery.HaveBucket) + if err != nil { + t.Fatal(err) + } + have := func(node, fp string) { + key, _ := sdkdelivery.HaveKey(node, "mesh-host") + raw, _ := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: fp, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}) + if _, err := haves.Put(ctx, key, raw); err != nil { + t.Fatal(err) + } + } + have("anchor", n6) + have("laptop", fpOf('5')) + if done, err := retireDeclarations(ctx, m.open); err != nil || done { + t.Fatalf("with laptop on the old node-engine: done %v (%v)", done, err) + } + if _, err := m.js.Stream(ctx, broker.DeclarationsStream); err != nil { + t.Fatalf("the stream went while laptop runs the old node-engine: %v", err) + } + have("laptop", n6) + if done, err := retireDeclarations(ctx, m.open); err != nil || !done { + t.Fatalf("with every node on the new node-engine: done %v (%v)", done, err) + } + if _, err := m.js.Stream(ctx, broker.DeclarationsStream); !errors.Is(err, jetstream.ErrStreamNotFound) { + t.Errorf("the stream is still there: %v", err) + } + if done, err := retireDeclarations(ctx, m.open); err != nil || !done { + t.Errorf("a second look: done %v (%v)", done, err) + } + if !declarationsRetiredNow(ctx, m.open.inventory) { + t.Errorf("a raise after the retirement would assert the stream again") + } +} diff --git a/cmd/mesh-controller/composed_judge.go b/cmd/mesh-controller/composed_judge.go new file mode 100644 index 00000000..e4d9be46 --- /dev/null +++ b/cmd/mesh-controller/composed_judge.go @@ -0,0 +1,343 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "path" + "sort" + "strings" + + sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" +) + +// The judgement of a move (novox/hq ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests): beside each composition's digest +// the controller writes whether moving the node from the composition it runs, as its have names it, to this one +// recreates a container, moves the module's data, or neither, or that it cannot be judged. The assigner holds a move of +// a module of policy record from it until a person allows it. +// +// ADR 0377 §3 over two compositions of one module: **recreates** when a container resource is changed, added or +// removed; **moves data** when, with no container changed, a directory or file resource is changed, added or removed +// that a container of the module mounts or that holds or lies within one of its directories (where a module keeps its +// data), when its `data` declaration changes, or, for a module that declares data, when one of its long-running +// processes changes; **neither** otherwise, and for a module the node has never run. **Cannot judge** when the +// composition the node runs is gone or cannot be read. A held composition never becomes what is judged against: with A +// running, B held and C composed, C is judged against A, the one the have names. + +// judgeMove is the value written beside a composition whose digest is digest: its judgement against what the node's have +// names for the module, and the module's policy. +func judgeMove(ctx context.Context, at composedPlace, key string, to sdkdelivery.Composed, digest string) (sdkdelivery.ComposedDigest, error) { + out := sdkdelivery.ComposedDigest{Digest: digest} + if at.policy != nil && to.Module != "" { + out.Policy, out.PolicyFrom, out.PolicyWhy = at.policy(to.Module) + } + if to.Module == "" { + // The node's own composition: no module's containers or data are judged; its want is written as any. + out.Judgement = sdkdelivery.JudgedNeither + return out, nil + } + have, found, err := at.ran(ctx, to.Module) + if err != nil { + return out, err + } + // A node whose have names no composition yet (the switch: it runs the declaration it last applied) is judged + // against that module's slice of that declaration (novox/hq ADR 0540 clause 9): equal is unchanged, a want that + // changes nothing on the node; otherwise by ADR 0377 §3's tests, as any move. + if found && have.Composed == "" && have.FP != "" && have.Phase != sdkdelivery.PhaseRemoved && at.lastApplied != nil { + slice, declared, applied, err := at.lastApplied(ctx, to.Module) + if err != nil { + // Unread is not judged: a person's yes holds it, and the words say why. + out.Judgement = sdkdelivery.JudgedCannot + out.Words = fmt.Sprintf("%s's have on %s names no composition, and the declaration it last applied cannot be read: %v", + to.Module, at.node, err) + return out, nil + } + if applied { + from := sdkdelivery.Composed{Node: at.node, Module: to.Module, FP: have.FP, Data: to.Data, Resources: slice} + if sameResources(from, to) { + out.Judgement, out.Words = sdkdelivery.JudgedUnchanged, "equal to the declaration "+at.node+" last applied" + return out, nil + } + // Against names the declaration, by its digest: the composition the node runs is that declaration's slice. + out.Against = declared + if onlyTheUserList(from, to) { + // Only the bus's user list, as against a composition (ADR 0539 clause 7): no yes waits on it at the switch. + out.Judgement, out.UserListOnly = sdkdelivery.JudgedNeither, true + out.Words = "only the bus's user list changes, against the declaration " + at.node + " last applied" + return out, nil + } + j, words := judged(from, to) + out.Judgement, out.Words = j, words+", against the declaration "+at.node+" last applied" + return out, nil + } + } + if !found || have.Phase == sdkdelivery.PhaseRemoved || (have.Composed == "" && have.FP == "") { + out.Judgement, out.Words = sdkdelivery.JudgedNeither, "the node runs no composition of "+to.Module + return out, nil + } + if have.Composed == "" { + out.Judgement = sdkdelivery.JudgedCannot + out.Words = fmt.Sprintf("%s's have on %s names no composition, only the fingerprint %s", to.Module, at.node, have.FP) + return out, nil + } + out.Against = have.Composed + if have.Composed == digest { + out.Judgement, out.Words = sdkdelivery.JudgedNeither, "the node runs this composition" + return out, nil + } + from, found, err := composedByDigest(ctx, at, to.Module, have.FP, have.Composed) + if err != nil { + return out, err + } + if !found { + out.Judgement = sdkdelivery.JudgedCannot + out.Words = fmt.Sprintf("the composition %s runs on %s (%s) is gone", to.Module, at.node, short(strings.TrimPrefix(have.Composed, "sha256:"))) + return out, nil + } + if onlyTheUserList(from, to) { + // Only the bus's user list (ADR 0539 clause 7): a file change and a reload of the bus, no move of it, written + // into its want at once. + out.Judgement, out.UserListOnly, out.Words = sdkdelivery.JudgedNeither, true, "only the bus's user list changes" + return out, nil + } + out.Judgement, out.Words = judged(from, to) + return out, nil +} + +// busUsersSuffix ends the id of the bus module's user list as the composer writes it: `.bus-users`. +const busUsersSuffix = ".bus-users" + +// onlyTheUserList says two compositions of a module differ, and only in its file of the bus's user list. +func onlyTheUserList(from, to sdkdelivery.Composed) bool { + if from.Data != to.Data { + return false + } + before, after := resourcesByID(from), resourcesByID(to) + if len(before) != len(after) { + return false + } + changed := false + for id, now := range after { + was, ok := before[id] + if !ok { + return false + } + if bytes.Equal(was.raw, now.raw) { + continue + } + if !strings.HasSuffix(id, busUsersSuffix) || now.kind != string(sdkdecl.TypeFile) { + return false + } + changed = true + } + return changed +} + +// composedByDigest finds, in the node's bucket, the composition of module at fingerprint fp whose digest is digest: the +// key's current value or one of the revisions it keeps. +func composedByDigest(ctx context.Context, at composedPlace, module, fp, digest string) (sdkdelivery.Composed, bool, error) { + if fp == "" { + return sdkdelivery.Composed{}, false, nil + } + key, err := sdkdelivery.ComposedKey(at.node, module, fp) + if err != nil { + return sdkdelivery.Composed{}, false, nil + } + values, err := historyOf(ctx, at.bucket, key) + if err != nil { + return sdkdelivery.Composed{}, false, err + } + for _, v := range values { + c, err := sdkdelivery.DecodeComposed(v) + if err != nil { + continue + } + if d, err := sdkdelivery.ComposedDigestOf(c); err == nil && d == digest { + return c, true, nil + } + } + return sdkdelivery.Composed{}, false, nil +} + +// historyOf is every value a key keeps, the newest first; none when there is no such key. +func historyOf(ctx context.Context, kv composedBucketsKV, key string) ([][]byte, error) { + if h, ok := kv.(interface { + History(ctx context.Context, key string, opts ...jetstream.WatchOpt) ([]jetstream.KeyValueEntry, error) + }); ok { + entries, err := h.History(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + out := make([][]byte, 0, len(entries)) + for i := len(entries) - 1; i >= 0; i-- { + if entries[i].Operation() == jetstream.KeyValuePut { + out = append(out, entries[i].Value()) + } + } + return out, nil + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return [][]byte{e.Value()}, nil +} + +// judged applies ADR 0377 §3 to two compositions of one module. +func judged(from, to sdkdelivery.Composed) (sdkdelivery.Judgement, string) { + before, after := resourcesByID(from), resourcesByID(to) + if ids := changedOf(before, after, func(kind string) bool { return kind == string(sdkdecl.TypeContainer) }); len(ids) > 0 { + return sdkdelivery.JudgedRecreates, "it recreates " + strings.Join(ids, ", ") + } + if from.Data != to.Data { + return sdkdelivery.JudgedMovesData, "its data declaration changes" + } + places := dataPlaces(before, after) + held := func(kind string) bool { + return kind == string(sdkdecl.TypeDirectory) || kind == string(sdkdecl.TypeFile) + } + var moved []string + for _, id := range changedOf(before, after, held) { + r := before[id] + if r == nil { + r = after[id] + } + if withinPlaces(r.path, places) { + moved = append(moved, id) + } + } + if to.Data || from.Data { + moved = append(moved, changedOf(before, after, func(kind string) bool { return kind == string(sdkdecl.TypeProcess) })...) + } + if len(moved) > 0 { + sort.Strings(moved) + return sdkdelivery.JudgedMovesData, "it changes where its data is kept: " + strings.Join(moved, ", ") + } + return sdkdelivery.JudgedNeither, "it recreates no container and moves no data" +} + +// judgedResource is a resource as the judgement reads it: its kind, the path it acts on, what its containers mount, +// and its canonical bytes. +type judgedResource struct { + kind string + path string + mounts []string + raw []byte +} + +func resourcesByID(c sdkdelivery.Composed) map[string]*judgedResource { + out := map[string]*judgedResource{} + for _, raw := range c.Resources { + var head struct { + ID string `json:"id"` + Type string `json:"type"` + Path string `json:"path"` + // A container's volumes, `:[:]`. + Volumes []string `json:"volumes"` + } + if err := json.Unmarshal(raw, &head); err != nil { + continue + } + r := &judgedResource{kind: head.Type, path: head.Path, raw: compactComposed(raw)} + for _, v := range head.Volumes { + source, _, _ := strings.Cut(v, ":") + r.mounts = append(r.mounts, source) + } + out[head.ID] = r + } + return out +} + +// compactComposed is a resource's bytes canonical: its keys sorted, its spacing dropped, its numbers as written, so +// two compositions that say the same compare equal however each was written. +func compactComposed(raw []byte) []byte { + var v any + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + if err := d.Decode(&v); err != nil { + return raw + } + out, err := json.Marshal(v) + if err != nil { + return raw + } + return out +} + +// changedOf are the ids of the resources of the kinds counted that are changed, added or removed, sorted. +func changedOf(before, after map[string]*judgedResource, counted func(kind string) bool) []string { + var out []string + for id, now := range after { + if !counted(now.kind) { + continue + } + if was, ok := before[id]; !ok || !bytes.Equal(was.raw, now.raw) { + out = append(out, id) + } + } + for id, was := range before { + if _, ok := after[id]; !ok && counted(was.kind) { + out = append(out, id) + } + } + sort.Strings(out) + return out +} + +// dataPlaces are the paths where the module keeps what a move must not lose: what its containers mount, and its +// directories, before and after. +func dataPlaces(before, after map[string]*judgedResource) []string { + var out []string + for _, rs := range []map[string]*judgedResource{before, after} { + for _, r := range rs { + if r.kind == string(sdkdecl.TypeContainer) { + out = append(out, r.mounts...) + } + if r.kind == string(sdkdecl.TypeDirectory) && r.path != "" { + out = append(out, r.path) + } + } + } + return out +} + +// withinPlaces says p is one of places, or lies within one, or holds one. +func withinPlaces(p string, places []string) bool { + if p == "" { + return false + } + p = path.Clean(p) + for _, place := range places { + if place == "" || !strings.HasPrefix(place, "/") { + continue + } + place = path.Clean(place) + if p == place || strings.HasPrefix(p, place+"/") || strings.HasPrefix(place, p+"/") { + return true + } + } + return false +} + +// sameResources says two compositions of a module hold the same resources in the same order, each compared canonical. +func sameResources(a, b sdkdelivery.Composed) bool { + if len(a.Resources) != len(b.Resources) { + return false + } + for i := range a.Resources { + if !bytes.Equal(compactComposed(a.Resources[i]), compactComposed(b.Resources[i])) { + return false + } + } + return true +} diff --git a/cmd/mesh-controller/composed_judge_test.go b/cmd/mesh-controller/composed_judge_test.go new file mode 100644 index 00000000..7316ae44 --- /dev/null +++ b/cmd/mesh-controller/composed_judge_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-controller/internal/link" +) + +// mailOn is a composition of mesh-mail on ace at fingerprint c, from the given resources, each saying its module. +func mailOn(t *testing.T, c byte, data bool, resources ...string) sdkdelivery.Composed { + t.Helper() + out := sdkdelivery.Composed{Node: "ace", Module: "mesh-mail", FP: sdkdelivery.SourcePrefix2 + strings.Repeat(string(c), 64), Data: data} + for _, r := range resources { + out.Resources = append(out.Resources, json.RawMessage(r)) + } + return out +} + +const ( + mailContainer = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:1111111111111111111111111111111111111111111111111111111111111111","volumes":["/var/lib/mesh-mail/spool:/spool"]}` + mailContainer2 = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:2222222222222222222222222222222222222222222222222222222222222222","volumes":["/var/lib/mesh-mail/spool:/spool"]}` + mailSpool = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/var/lib/mesh-mail/spool"}` + mailSpoolMoved = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/srv/mesh-mail/spool"}` + mailTools = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"a"}` + mailTools2 = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"b"}` + mailConfig = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"a"}` + mailConfig2 = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"b"}` +) + +// 1945/AC12: captured pairs of compositions are judged by ADR 0377 §3's tests: a changed container recreates; with no +// container changed, a directory moved, a file within the data a container mounts changed, or the data declaration +// changed moves data; a tools file nothing mounts changed is neither. +func TestAMoveIsJudgedByWhatItRecreatesOrMoves(t *testing.T) { + cases := []struct { + name string + from, to sdkdelivery.Composed + want sdkdelivery.Judgement + }{ + {"a new image", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates}, + {"a container gone", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailSpool), sdkdelivery.JudgedRecreates}, + {"its spool moved", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpoolMoved), sdkdelivery.JudgedMovesData}, + {"a file in its spool", mailOn(t, 'a', true, mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData}, + {"its data declaration", mailOn(t, 'a', false, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpool), sdkdelivery.JudgedMovesData}, + {"its tools only", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither}, + } + for _, c := range cases { + if got, words := judged(c.from, c.to); got != c.want || words == "" { + t.Errorf("%s: judged %s (%q), want %s", c.name, got, words, c.want) + } + } +} + +// 1945/AC12: a move is judged against the composition the node's have names: one the node never ran is neither; one +// whose composition is gone cannot be judged and says why; and with A running, B held and C composed, C is judged +// against A, and judged again when the have changes. +func TestAMoveIsJudgedAgainstWhatTheNodeRuns(t *testing.T) { + at, _, _, haves := aPlace(t, "ace") + ctx := context.Background() + a := mailOn(t, 'a', true, mailContainer, mailSpool, mailTools) + b := mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools) + c := mailOn(t, 'c', true, mailContainer, mailSpool, mailTools2) + write := func(comp sdkdelivery.Composed) (string, string) { + key, _ := sdkdelivery.ComposedKey("ace", comp.Module, comp.FP) + if _, err := writeComposed(ctx, at, key, comp); err != nil { + t.Fatal(err) + } + d, _ := sdkdelivery.ComposedDigestOf(comp) + return key, d + } + judgedNow := func(key string) sdkdelivery.ComposedDigest { + e, err := at.digests.Get(ctx, key) + if err != nil { + t.Fatal(err) + } + v, err := sdkdelivery.DecodeComposedDigest(e.Value()) + if err != nil { + t.Fatal(err) + } + return v + } + keyA, digestA := write(a) + if v := judgedNow(keyA); v.Judgement != sdkdelivery.JudgedNeither || v.Against != "" { + t.Errorf("a module the node never ran: %+v", v) + } + if v := judgedNow(keyA); v.Policy != "roll" || v.PolicyFrom != "default" { + t.Errorf("the policy beside the judgement: %+v", v) + } + haves["mesh-mail"] = sdkdelivery.Have{FP: a.FP, Composed: digestA, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + keyB, _ := write(b) + if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedRecreates || v.Against != digestA { + t.Errorf("B against A: %+v", v) + } + // B is held (the assigner writes no want for it); the node still runs A. + keyC, digestC := write(c) + if v := judgedNow(keyC); v.Judgement != sdkdelivery.JudgedNeither || v.Against != digestA { + t.Errorf("C against A, B held: %+v", v) + } + // The have changes to C: every judgement of the module is written again, against what runs now (the have watch). + haves["mesh-mail"] = sdkdelivery.Have{FP: c.FP, Composed: digestC, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + if n, err := rejudge(ctx, at, "mesh-mail"); err != nil || n == 0 { + t.Fatalf("rejudged %d (%v)", n, err) + } + if v := judgedNow(keyC); v.Against != digestC || v.Judgement != sdkdelivery.JudgedNeither { + t.Errorf("C once the node runs it: %+v", v) + } + if v := judgedNow(keyB); v.Against != digestC || v.Judgement != sdkdelivery.JudgedRecreates { + t.Errorf("B once the node runs C: %+v", v) + } + // A person sets the module's policy: written beside every composition of it. + at.policy = func(string) (string, string, string) { return "record", "person", "its spool" } + if _, err := rejudge(ctx, at, "mesh-mail"); err != nil { + t.Fatal(err) + } + if v := judgedNow(keyB); v.Policy != "record" || v.PolicyFrom != "person" || v.PolicyWhy != "its spool" { + t.Errorf("the person's policy beside B: %+v", v) + } + // A have naming a composition the buckets no longer hold. + haves["mesh-mail"] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("9", 64), Composed: "sha256:" + strings.Repeat("9", 64), + Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + write(b) + if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedCannot || !strings.Contains(v.Words, "gone") { + t.Errorf("against a composition that is gone: %+v", v) + } +} + +const ( + busServer = `{"id":"nats.server","type":"container","module":"nats","name":"nats","image":"nats@sha256:4444444444444444444444444444444444444444444444444444444444444444"}` + busServer2 = `{"id":"nats.server","type":"container","module":"nats","name":"nats","image":"nats@sha256:5555555555555555555555555555555555555555555555555555555555555555"}` + busUsers = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users.conf","content":"users: [a]"}` + busUsers2 = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users.conf","content":"users: [a, b]"}` +) + +// busOn is a composition of the bus on ace from the given resources. +func busOn(t *testing.T, resources ...string) sdkdelivery.Composed { + t.Helper() + out := sdkdelivery.Composed{Node: "ace", Module: "nats", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64)} + for _, r := range resources { + out.Resources = append(out.Resources, json.RawMessage(r)) + } + return out +} + +// ADR 0540 clause 9: a have that names no composition, only a fingerprint (the switch: the node runs the declaration it +// last applied), is judged against the module's slice of that declaration, read from the kept stream of declarations: +// an equal slice is unchanged; a slice differing only in the bus's user list is that (round 1's A2, so no yes waits on it at the switch); a changed setting is judged as any move; no declaration to read cannot be judged. +func TestAHaveWithNoCompositionIsJudgedAgainstTheLastDeclaration(t *testing.T) { + ctx := context.Background() + other := `{"id":"mesh-calendar.server","type":"container","module":"mesh-calendar","name":"calendar","image":"cal@sha256:3333333333333333333333333333333333333333333333333333333333333333"}` + unsaid := `{"id":"mesh-mail.tools","type":"file","path":"/usr/lib/mesh-mail/tools.json","content":"a"}` + declared := func(resources ...string) []byte { + body, err := json.Marshal(map[string]any{"node": "ace", "resources": json.RawMessage("[" + strings.Join(resources, ",") + "]")}) + if err != nil { + t.Fatal(err) + } + signed, err := json.Marshal(link.Signed{Declaration: body, Signature: []byte("s")}) // as link.Envelope sent it + if err != nil { + t.Fatal(err) + } + return signed + } + cases := []struct { + name string + last []byte // the node's last declaration on the stream; nil when there is none + to sdkdelivery.Composed + want sdkdelivery.Judgement + inWords string + userList bool + }{ + {"an equal slice", declared(mailContainer, other, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedUnchanged, "last applied", false}, + {"a declared resource saying no module is not equal", declared(mailContainer, other, mailSpool, unsaid), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedNeither, "last applied", false}, + {"a changed image", declared(mailContainer, other, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates, "last applied", false}, + {"a changed setting in its spool", declared(mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData, "last applied", false}, + {"a changed tools file", declared(mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither, "last applied", false}, + {"only the bus's user list", declared(busServer, busUsers, other), busOn(t, busServer, busUsers2), sdkdelivery.JudgedNeither, "only the bus's user list", true}, + {"the bus's user list and its container", declared(busServer, busUsers), busOn(t, busServer2, busUsers2), sdkdelivery.JudgedRecreates, "last applied", false}, + {"no declaration kept", nil, mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedCannot, "names no composition", false}, + } + for _, c := range cases { + at, _, _, haves := aPlace(t, "ace") + last := c.last + at.lastApplied = func(_ context.Context, module string) ([]json.RawMessage, string, bool, error) { + if last == nil { + return nil, "", false, nil + } + return sliceOfDeclaration(last, module) + } + haves[c.to.Module] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + key, _ := sdkdelivery.ComposedKey("ace", c.to.Module, c.to.FP) + digest, err := sdkdelivery.ComposedDigestOf(c.to) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + got, err := judgeMove(ctx, at, key, c.to, digest) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if got.UserListOnly != c.userList { + t.Errorf("%s: user list only %v, want %v", c.name, got.UserListOnly, c.userList) + } + if got.Judgement != c.want || !strings.Contains(got.Words, c.inWords) { + t.Errorf("%s: judged %s (%q), want %s", c.name, got.Judgement, got.Words, c.want) + } + if _, err := sdkdelivery.EncodeComposedDigest(got); err != nil { + t.Errorf("%s: the judgement written is refused: %v", c.name, err) + } + } +} diff --git a/cmd/mesh-controller/composed_keys.go b/cmd/mesh-controller/composed_keys.go new file mode 100644 index 00000000..2a95d9ea --- /dev/null +++ b/cmd/mesh-controller/composed_keys.go @@ -0,0 +1,264 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A new fingerprint of a registered module, from its done key to its compositions (novox/hq ADR 0539 clauses 5 and 7, +// ADR 0501 §2). With no delivery plan left to take a module's done key in, a write of the builder's or the indexer's +// keys does it: the controller takes in the done key of the fingerprint the indexer's trunk value names (registers the +// module at it, as a plan's take-in did), composes every node the module is assigned to (which makes what a module's +// new manifest needs, a credential among it, as a send made it), composes the node holding the bus (its user list now +// carries the module's permissions at the new fingerprint: a new revision of the bus module's composition alone, +// written into its want at once by the assigner), and writes `grants..`. The assigner's sixth and seventh +// rules then let a want name the fingerprint. **No timer**: woken by the keys, and a starting controller looks at every +// module once (B5's start pass). + +// doneKeySteps are what taking a module's done key in does, each a step a test may stand in for. +type doneKeySteps struct { + // held says the catalogue holds the module (a new one is the new-module take-in's, new_modules.go). + held func(ctx context.Context, module string) (bool, error) + // trunk is the indexer's trunk value of the module. + trunk func(ctx context.Context, module string) (sdkdelivery.Module, error) + // registeredAt says the module is registered at the commit. + registeredAt func(ctx context.Context, module, commit string) (bool, error) + // takeIn registers the module from the builder's done key of the trunk's fingerprint (takeInKey); errKeyWaits when + // the key is not done yet. + takeIn func(ctx context.Context, module string, trunk sdkdelivery.Module) error + // nodesOf are the nodes the module is assigned to; busHolder the node holding the bus, "" for none. + nodesOf func(ctx context.Context, module string) ([]string, error) + busHolder func(ctx context.Context) (string, error) + // compose writes the nodes' compositions; grants writes the grants issued. + compose func(ctx context.Context, nodes []string) error + grants func(ctx context.Context) error +} + +// take takes module's done key in and composes what it changed. It answers what it did in a line, "" when there was +// nothing to do. +func (s doneKeySteps) take(ctx context.Context, module string) (string, error) { + held, err := s.held(ctx, module) + if err != nil || !held { + return "", err + } + trunk, err := s.trunk(ctx, module) + if errors.Is(err, builderkeys.ErrNoValue) { + return "", nil + } + if err != nil { + return "", err + } + if !strings.HasPrefix(trunk.FP2, sdkdelivery.SourcePrefix2) { + return "", nil + } + at, err := s.registeredAt(ctx, module, trunk.Commit) + if err != nil { + return "", err + } + said := "" + if !at { + if err := s.takeIn(ctx, module, trunk); err != nil { + return "", err + } + said = fmt.Sprintf("%s registered from the builder's done key %s (%s)", module, short(trunk.FP2), short(trunk.Commit)) + } + nodes, err := s.nodesOf(ctx, module) + if err != nil { + return said, err + } + if len(nodes) == 0 { + return said, nil + } + // The module's nodes first: composing them makes what the new manifest needs, its credentials among it. + if err := s.compose(ctx, nodes); err != nil { + return said, err + } + // Then the bus's user list, which carries the module's permissions at its new fingerprint. + holder, err := s.busHolder(ctx) + if err != nil { + return said, err + } + if holder != "" && !inList(nodes, holder) { + if err := s.compose(ctx, []string{holder}); err != nil { + return said, err + } + } + // Then the grant, which with the composition lets a want name the fingerprint (rules 6 and 7). + if err := s.grants(ctx); err != nil { + return said, err + } + if said == "" { + said = fmt.Sprintf("%s at %s", module, short(trunk.FP2)) + } + return said + "; composed for " + strings.Join(nodes, ", ") + ", its grant written", nil +} + +func inList(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// doneKeysMu keeps one take-in of a module's done key at a time in this process, per module. +var doneKeysMu sync.Map + +// takeDoneKey is a woken module's take-in on the live stores and buckets, said. A take-in that fails (its compositions +// or its grant not written) is tried again every doneKeyRetry while it fails, loudly: printed each time, and an urgent +// condition raised by the first until it has worked, as the chain's own moves are (layBuildsOutSoon). A key not done +// yet is no failure: its next write wakes this again. +func takeDoneKey(ctx context.Context, open *stores, module string) { + mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + defer mu.(*sync.Mutex).Unlock() + failed := false + for { + err := takeDoneKeyOnce(ctx, open, module) + o := doneKeyObservation(module) + if err == nil || errors.Is(err, errKeyWaits) { + if keeper := conditionsFrom; keeper != nil && failed { + _, _ = keeper.Clear(ctx, o.Key(), "composed and granted") + } + return + } + fmt.Printf("done keys: %s: %v; tried again in %s\n", module, err, doneKeyRetry) + if keeper := conditionsFrom; keeper != nil && !failed { + o.Summary += err.Error() + if _, oerr := keeper.Observe(ctx, o); oerr == nil { + failed = true + } + } + select { + case <-ctx.Done(): + return + case <-time.After(doneKeyRetry): + } + } +} + +// doneKeyRetry is how long a failed take-in of a done key waits before it tries again. +var doneKeyRetry = 30 * time.Second + +// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until +// they are, no want can name it. +const kindNotComposed = "fingerprint-not-composed" + +func doneKeyObservation(module string) conditions.Observation { + return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed, + Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys", + Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "} +} + +// takeDoneKeyOnce is one take-in of a module's done key. +func takeDoneKeyOnce(ctx context.Context, open *stores, module string) error { + keys, closeKeys, err := openBuilderKeys() + if err != nil { + return fmt.Errorf("the builder's keys cannot be read: %w", err) + } + defer closeKeys() + said, err := liveDoneKeySteps(open, keys).take(ctx, module) + if err == nil && said != "" { + fmt.Printf("done keys: %s\n", said) + } + return err +} + +// liveDoneKeySteps are the steps on the controller's own stores and the bus. +func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps { + inv := open.inventory + return doneKeySteps{ + held: func(ctx context.Context, module string) (bool, error) { + _, err := inv.SourceOf(ctx, module) + if errors.Is(err, inventory.ErrNoSuchModule) { + return false, nil + } + return err == nil, err + }, + trunk: func(ctx context.Context, module string) (sdkdelivery.Module, error) { + return keys.ModuleAt(ctx, module, "") + }, + registeredAt: func(ctx context.Context, module, commit string) (bool, error) { + src, err := inv.SourceOf(ctx, module) + if err != nil { + return false, err + } + return sameCommitID(src.BuiltFrom, commit), nil + }, + takeIn: func(ctx context.Context, module string, trunk sdkdelivery.Module) error { + _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if errors.Is(err, inventory.ErrSuperseded) { + return nil + } + return err + }, + nodesOf: func(ctx context.Context, module string) ([]string, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, err + } + for _, e := range entries { + if e.Manifest.Module == module { + return e.On, nil + } + } + return nil, nil + }, + busHolder: func(ctx context.Context) (string, error) { + holders, err := seatHolders(ctx, inv) + if err != nil { + return "", err + } + return holders[catalogue.BrokerSeat].Node, nil + }, + compose: func(ctx context.Context, nodes []string) error { + _, err := composeModules(ctx, open, nodes) + return err + }, + grants: func(ctx context.Context) error { + bus, err := aBus() + if err != nil { + return err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return err + } + return grantsIssued(ctx, inv, js, "done keys: ") + }, + } +} + +// watchDoneKeys wakes the take-in of a module's done key for each module the builder or the indexer writes, until ctx +// ends: never on the watch's own goroutine. +func watchDoneKeys(ctx context.Context, open *stores) { + keys, closeKeys, err := openBuilderKeys() + if err != nil { + fmt.Printf("done keys: the builder's keys are not watched: %v\n", err) + return + } + if err := builderkeys.Watch(ctx, keys.JS, func(module string) { + inBackground(func() { takeDoneKey(ctx, open, module) }) + }); err != nil { + closeKeys() + fmt.Printf("done keys: the builder's keys are not watched: %v\n", err) + return + } + go func() { + <-ctx.Done() + closeKeys() + }() +} diff --git a/cmd/mesh-controller/composed_keys_test.go b/cmd/mesh-controller/composed_keys_test.go new file mode 100644 index 00000000..d29be2c9 --- /dev/null +++ b/cmd/mesh-controller/composed_keys_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" +) + +// stepsFor are doneKeySteps over a module registered at registered, whose trunk value names fp at commit, assigned to +// nodes, the bus held on holder; every act is written to acts in order. +func stepsFor(acts *[]string, registered, commit, fp string, nodes []string, holder string) doneKeySteps { + return doneKeySteps{ + held: func(_ context.Context, module string) (bool, error) { return module == "mesh-mail", nil }, + trunk: func(context.Context, string) (sdkdelivery.Module, error) { + if fp == "" { + return sdkdelivery.Module{}, builderkeys.ErrNoValue + } + return sdkdelivery.Module{Commit: commit, FP2: fp}, nil + }, + registeredAt: func(_ context.Context, _ string, c string) (bool, error) { return c == registered, nil }, + takeIn: func(_ context.Context, module string, trunk sdkdelivery.Module) error { + *acts = append(*acts, "take in "+module+" at "+trunk.Commit) + return nil + }, + nodesOf: func(context.Context, string) ([]string, error) { return nodes, nil }, + busHolder: func(context.Context) (string, error) { return holder, nil }, + compose: func(_ context.Context, ns []string) error { + *acts = append(*acts, "compose "+strings.Join(ns, ",")) + return nil + }, + grants: func(context.Context) error { *acts = append(*acts, "grants"); return nil }, + } +} + +// 1945/AC3 (unit half): a done key of a new fingerprint of an assigned module is taken in, then the module's nodes are +// composed, then the node holding the bus (its user list), then the grants: the grant and the composition exist +// before the assigner's rules let a want name the fingerprint. A module registered at the trunk's commit already is +// composed and granted without a second take-in; one assigned nowhere, unregistered, or with no trunk value is left +// alone. +func TestADoneKeyIsTakenInComposedAndGranted(t *testing.T) { + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64) + var acts []string + said, err := stepsFor(&acts, "old", "new", fp, []string{"ace", "g14"}, "novox").take(context.Background(), "mesh-mail") + if err != nil { + t.Fatal(err) + } + if got := strings.Join(acts, "; "); got != "take in mesh-mail at new; compose ace,g14; compose novox; grants" { + t.Errorf("the acts were %q", got) + } + if !strings.Contains(said, "registered from the builder's done key") { + t.Errorf("said %q", said) + } + + acts = nil + if _, err := stepsFor(&acts, "new", "new", fp, []string{"novox"}, "novox").take(context.Background(), "mesh-mail"); err != nil { + t.Fatal(err) + } + if got := strings.Join(acts, "; "); got != "compose novox; grants" { + t.Errorf("a module registered at its trunk already: %q", got) + } + + for name, s := range map[string]doneKeySteps{ + "assigned nowhere": stepsFor(&acts, "old", "new", fp, nil, "novox"), + "no trunk value": stepsFor(&acts, "old", "new", "", []string{"ace"}, "novox"), + } { + acts = nil + if _, err := s.take(context.Background(), "mesh-mail"); err != nil { + t.Fatal(err) + } + if name == "assigned nowhere" && strings.Join(acts, "; ") != "take in mesh-mail at new" { + t.Errorf("%s: %v", name, acts) + } + if name == "no trunk value" && len(acts) != 0 { + t.Errorf("%s: %v", name, acts) + } + } + acts = nil + if _, err := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox").take(context.Background(), "unknown"); err != nil || len(acts) != 0 { + t.Errorf("a module the catalogue does not hold: %v %v", acts, err) + } +} + +// 1945/AC4: the bus's user list is a file of the bus module's composition (ADR 0539 clause 7): a change of the composed +// users is a new revision of that composition alone, every other composition of the node keeping its digest. +func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testing.T) { + body, fps := capturedBody(t, "g14") + fps["nats"] = sdkdelivery.SourcePrefix2 + strings.Repeat("e", 64) + withUsers := func(users string) map[string]sdkdelivery.Composed { + var d map[string]any + if err := json.Unmarshal(body, &d); err != nil { + t.Fatal(err) + } + // As the composer writes it for the module holding mesh-broker (mesh-sdk catalogue: BusUsersID, owned by the module). + d["resources"] = append(d["resources"].([]any), map[string]any{"id": "nats.bus-users", "type": "file", + "module": "nats", "path": "/var/lib/mesh/nats/users.conf", "content": users, "mode": "0600"}) + raw, _ := json.Marshal(d) + comps, _, err := compositionsOf("g14", raw, fps, nil, nil) + if err != nil { + t.Fatal(err) + } + return comps + } + before, after := withUsers("authorization { users = [ {user: a} ] }\n"), withUsers("authorization { users = [ {user: a}, {user: b} ] }\n") + natsKey, _ := sdkdelivery.ComposedKey("g14", "nats", fps["nats"]) + for key, c := range before { + d1, _ := sdkdelivery.ComposedDigestOf(c) + d2, _ := sdkdelivery.ComposedDigestOf(after[key]) + if changed := d1 != d2; changed != (key == natsKey) { + t.Errorf("%s changed %v; only the bus module's composition changes with its user list", key, changed) + } + } +} + +// 1945 (the review's D1): a grant that is not written is the take-in's error, never swallowed: with no send left, +// nothing else would write it, and no want could name the fingerprint. +func TestAGrantNotWrittenIsTheTakeInsError(t *testing.T) { + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64) + var acts []string + s := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox") + s.grants = func(context.Context) error { return errors.New("the bucket mesh-controller_grants: not found") } + if _, err := s.take(context.Background(), "mesh-mail"); err == nil || !strings.Contains(err.Error(), "grants") { + t.Errorf("a grant not written: %v", err) + } +} diff --git a/cmd/mesh-controller/composed_live.go b/cmd/mesh-controller/composed_live.go new file mode 100644 index 00000000..a1ea3ead --- /dev/null +++ b/cmd/mesh-controller/composed_live.go @@ -0,0 +1,362 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" +) + +// composeModules composes each named node and writes its compositions that differ (ADR 0539 clause 5): what each act +// that once sent a declaration calls instead. A node that cannot compose is that node's: the others are written, and +// every refusal is named in the error. It answers the keys it wrote, by node. Under the lease only: a controller that +// does not hold it writes nothing and says so. +func composeModules(ctx context.Context, open *stores, nodes []string) (map[string][]string, error) { + if _, err := theLease.epoch(ctx); err != nil { + return nil, fmt.Errorf("nothing is composed: %w", err) + } + bus, err := aBus() + if err != nil { + return nil, fmt.Errorf("nothing is composed: the bus: %w", err) + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return nil, err + } + sign, err := controllerSigner(ctx, open) + if err != nil { + return nil, fmt.Errorf("nothing is composed: the controller's signing key: %w", err) + } + fps, err := currentFingerprints(ctx, open.inventory) + if err != nil { + return nil, err + } + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + return nil, err + } + manifests := make(map[string]catalogue.Manifest, len(entries)) + for _, e := range entries { + manifests[e.Manifest.Module] = e.Manifest + } + upgrades, err := open.inventory.Upgrades(ctx) + if err != nil { + return nil, err + } + policy := func(module string) (string, string, string) { + u, ok := upgrades[module] + if !ok { + return "", "", "" + } + return u.Policy(), u.From, u.Why + } + sort.Strings(nodes) + out := map[string][]string{} + var errs []error + for _, node := range nodes { + at, err := openComposedPlace(ctx, js, bus.Conn(), node) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", node, err)) + continue + } + at.sign, at.policy = sign, policy + if composeTrace != nil { + traceInput(ctx, open, node) + } + declared, err := composeSend(ctx, open, node) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", node, err)) + continue + } + body, err := declared.Body() + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", node, err)) + continue + } + // The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror. + if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil { + errs = append(errs, fmt.Errorf("%s: its provisioner's plans: %w", node, err)) + } + comps, moved, err := compositionsOf(node, body, fps, manifests, declared.modules) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", node, err)) + continue + } + if len(moved) > 0 { + fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n", + node, strings.Join(moved, ", ")) + } + if composeTrace != nil { + traceComposed(ctx, at, node, comps) + } + wrote, err := writeNodeCompositions(ctx, at, comps) + if len(wrote) > 0 { + out[node] = wrote + fmt.Printf("compositions: %s: wrote %s\n", node, strings.Join(wrote, ", ")) + } + // And where a data-keeping provision bound each consumer: what the next resolution keeps it at (novox/hq ADR 0232). + if len(declared.Bindings) > 0 { + if err := open.inventory.RecordBindings(ctx, node, declared.Bindings); err != nil { + errs = append(errs, fmt.Errorf("%s: where its consumers are bound to their data: %w", node, err)) + } + } + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", node, err)) + } + } + return out, errors.Join(errs...) +} + +// composeAndSay is composeModules for an act that has nothing to do with its answer but say it: what each former send +// of a verb or an event calls. +func composeAndSay(ctx context.Context, open *stores, nodes []string) error { + if len(nodes) == 0 { + return nil + } + _, err := composeModules(ctx, open, nodes) + if err != nil { + fmt.Printf("compositions: %v\n", err) + } + return err +} + +// openComposedPlace opens a node's buckets: its compositions, the digests, its files, and reads its want and have. +func openComposedPlace(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, node string) (composedPlace, error) { + if err := broker.EnsureComposed(ctx, js, node); err != nil { + return composedPlace{}, err + } + name, err := sdkdelivery.ComposedBucketFor(node) + if err != nil { + return composedPlace{}, err + } + bucket, err := js.KeyValue(ctx, name) + if err != nil { + return composedPlace{}, fmt.Errorf("the bucket %s: %w", name, err) + } + digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket) + if err != nil { + return composedPlace{}, fmt.Errorf("the bucket %s: %w", sdkdelivery.ControllerComposedBucket, err) + } + objects, err := openBucket(ctx, conn, node) + if err != nil { + return composedPlace{}, err + } + return composedPlace{node: node, bucket: bucket, digests: digests, objects: objects, + wanted: func(ctx context.Context, module string) (string, error) { return wantedOn(ctx, js, node, module) }, + ran: func(ctx context.Context, module string) (sdkdelivery.Have, bool, error) { + return haveOn(ctx, js, node, module) + }, + lastApplied: lastAppliedOn(js, node), + }, nil +} + +// wantedOn is the fingerprint the assigner's want names for a module on a node; "" when there is none or the bucket is +// not there yet. +func wantedOn(ctx context.Context, js jetstream.JetStream, node, module string) (string, error) { + kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return "", nil + } + if err != nil { + return "", err + } + key, err := sdkdelivery.WantKey(node, module) + if err != nil { + return "", err + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return "", nil + } + if err != nil { + return "", err + } + w, err := sdkdelivery.DecodeWant(e.Value()) + if err != nil { + return "", nil // a want of release 5, naming no composition: no fingerprint a composition must be kept for + } + return w.FP, nil +} + +// haveOn is what a node's node-engine says it runs of a module. +func haveOn(ctx context.Context, js jetstream.JetStream, node, module string) (sdkdelivery.Have, bool, error) { + kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return sdkdelivery.Have{}, false, nil + } + if err != nil { + return sdkdelivery.Have{}, false, err + } + key, err := sdkdelivery.HaveKey(node, module) + if err != nil { + return sdkdelivery.Have{}, false, err + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return sdkdelivery.Have{}, false, nil + } + if err != nil { + return sdkdelivery.Have{}, false, err + } + h, err := sdkdelivery.DecodeHave(e.Value()) + if err != nil { + return sdkdelivery.Have{}, false, nil + } + return h, true, nil +} + +// wantedOnTheBus is every fingerprint a want names, as ` `: what the keep set keeps the builds of +// (ADR 0539 clause 1). Read from the assigner's bucket on each look; no bucket yet names none. +func wantedOnTheBus(ctx context.Context) (map[string]bool, error) { + bus, err := aBus() + if err != nil { + return nil, err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return nil, err + } + kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return map[string]bool{}, nil + } + if err != nil { + return nil, err + } + lister, err := kv.ListKeys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return map[string]bool{}, nil + } + if err != nil { + return nil, err + } + out := map[string]bool{} + for key := range lister.Keys() { + _, module, err := sdkdelivery.ParseWantKey(key) + if err != nil { + continue + } + e, err := kv.Get(ctx, key) + if err != nil { + continue + } + w, err := sdkdelivery.DecodeWant(e.Value()) + if err != nil || w.FP == "" { + continue + } + out[module+" "+w.FP] = true + } + return out, nil +} + +// plansKept is a composition's provisioner's plans as the store keeps them: by module, each plan and what requires a +// module only pulled in by another (design 57 P6, issue 765); nil when it hands none. +func plansKept(d sendable) []byte { + if len(d.plans) == 0 { + return nil + } + kept := map[string]composedPlan{} + for module, p := range d.plans { + kept[module] = composedPlan{Plan: p, RequiredBy: d.requiredBy[module]} + } + raw, _ := json.Marshal(kept) // plain strings and slices: it marshals + return raw +} + +// composedPlan is one handed module's plan as the store keeps it, the shape inventory.SentProvisioningPlans reads. +type composedPlan struct { + catalogue.Plan + RequiredBy string `json:"required-by,omitempty"` +} + +// lastAppliedOn reads, once, the declaration the node was last sent from the kept stream of declarations (the last +// message on `mesh.node..declare`), and answers each module's slice of it (its resources that say the module, or +// whose id names it) with the declaration's digest. None when the stream or the node's message is gone. +func lastAppliedOn(js jetstream.JetStream, node string) func(ctx context.Context, module string) ([]json.RawMessage, string, bool, error) { + var once sync.Once + var last []byte + var found bool + var readErr error + return func(ctx context.Context, module string) ([]json.RawMessage, string, bool, error) { + once.Do(func() { last, found, readErr = lastDeclaration(ctx, js, node) }) + if readErr != nil || !found { + return nil, "", false, readErr + } + return sliceOfDeclaration(last, module) + } +} + +// lastDeclaration is the node's last signed declaration on the kept stream, as the stream holds it. +func lastDeclaration(ctx context.Context, js jetstream.JetStream, node string) ([]byte, bool, error) { + stream, err := js.Stream(ctx, broker.DeclarationsStream) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + msg, err := stream.GetLastMsgForSubject(ctx, "mesh.node."+node+".declare") + if errors.Is(err, jetstream.ErrMsgNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return msg.Data, true, nil +} + +// sliceOfDeclaration is module's slice of a signed declaration as the stream holds it, and the digest of the +// declaration it was signed over. +func sliceOfDeclaration(data []byte, module string) ([]json.RawMessage, string, bool, error) { + var signed link.Signed + if err := json.Unmarshal(data, &signed); err != nil { + return nil, "", false, fmt.Errorf("the last declaration is not readable: %w", err) + } + var envelope struct { + Resources []json.RawMessage `json:"resources"` + } + if err := json.Unmarshal(signed.Declaration, &envelope); err != nil { + return nil, "", false, fmt.Errorf("the last declaration is not readable: %w", err) + } + sum := sha256.Sum256(signed.Declaration) + return sliceOfModule(envelope.Resources, module), "sha256:" + hex.EncodeToString(sum[:]), true, nil +} + +// sliceOfModule is, of a declaration's resources in their order, those of module: saying it, or with no module said +// and an id that names it (sdkdecl.ModuleGuess). +func sliceOfModule(resources []json.RawMessage, module string) []json.RawMessage { + var out []json.RawMessage + for _, r := range resources { + var head struct { + ID string `json:"id"` + Module string `json:"module"` + } + if json.Unmarshal(r, &head) != nil { + continue + } + owner := head.Module + if owner == "" { + owner, _ = sdkdecl.ModuleGuess(head.ID) + } + if owner == module { + out = append(out, r) + } + } + return out +} diff --git a/cmd/mesh-controller/composed_operator_keys_test.go b/cmd/mesh-controller/composed_operator_keys_test.go new file mode 100644 index 00000000..f06a94ab --- /dev/null +++ b/cmd/mesh-controller/composed_operator_keys_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1945/AC13: the node-engine's composition on a node carries the operator's public keys as its file operator-keys, +// from its setting operator-signers, beside the controller's key it already trusts (ADR 0539 clause 3, decided with the +// lead 2026-10-24: the switch's last declaration carries it because today's controller composes the same file from +// the same manifest). The manifest is mesh-host's module.json at 20d0998c (testdata/mesh-host-module.json), never +// typed by hand. +func TestTheNodeEnginesCompositionCarriesTheOperatorsKeys(t *testing.T) { + raw, err := os.ReadFile("testdata/mesh-host-module.json") + if err != nil { + t.Fatal(err) + } + var m catalogue.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + // Its archive names a build artifact, which only its registration turns into a reference: left out here, the rest of + // the manifest as captured. + var kept []map[string]any + for _, r := range m.Resources { + if r["type"] != "archive" { + kept = append(kept, r) + } + } + m.Resources = kept + const key = "MCowBQYDK2VwAyEAb3BlcmF0b3Itc2lnbmluZy1rZXktZm9yLXRlc3RzLW9ubHk=" + in := compose.Input{Catalogue: map[string]catalogue.Manifest{m.Module: m}, + Nodes: []compose.Node{{Name: "ace", Assigned: []string{m.Module}, Stated: true, + Settings: map[string][]catalogue.Layer{m.Module: {{From: "mesh", Values: map[string]any{"operator-signers": key}}}}}}} + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64) + all, _, err := compose.Compositions(in, "ace", map[string]string{m.Module: fp}, compose.Reading) + if err != nil { + t.Fatal(err) + } + k, _ := sdkdelivery.ComposedKey("ace", m.Module, fp) + var file struct{ Path, Content string } + for _, r := range all[k].Resources { + var head struct{ ID, Path, Content string } + _ = json.Unmarshal(r, &head) + if strings.HasSuffix(head.ID, "operator-keys") { + file.Path, file.Content = head.Path, head.Content + } + } + if file.Path != "/var/lib/mesh-host/operator-keys" || file.Content != key+"\n" { + t.Errorf("the node-engine's composition carries operator-keys at %q with %q, want the operator's key", file.Path, file.Content) + } +} diff --git a/cmd/mesh-controller/composed_plans_test.go b/cmd/mesh-controller/composed_plans_test.go new file mode 100644 index 00000000..375b0d0e --- /dev/null +++ b/cmd/mesh-controller/composed_plans_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// 1945 (dev-r6-controller's finding): the provisioner's plans a composition hands are kept in the shape the +// assignments mirror reads (inventory.SentProvisioningPlans: each module's plan and what requires it), and a +// composition that hands none keeps none. +func TestAComposedNodesProvisionerPlansAreKept(t *testing.T) { + if plansKept(sendable{}) != nil { + t.Errorf("a composition that hands nothing kept plans") + } + d := sendable{plans: map[string]catalogue.Plan{"umami": {Placement: map[string]string{"dir:state": "/var/lib/umami"}}}, + requiredBy: map[string]string{"umami": "analytics"}} + var kept map[string]struct { + Placement map[string]string `json:"placement"` + RequiredBy string `json:"required-by"` + } + if err := json.Unmarshal(plansKept(d), &kept); err != nil { + t.Fatal(err) + } + if kept["umami"].Placement["dir:state"] != "/var/lib/umami" || kept["umami"].RequiredBy != "analytics" { + t.Errorf("kept %+v", kept) + } +} diff --git a/cmd/mesh-controller/composed_rejudge.go b/cmd/mesh-controller/composed_rejudge.go new file mode 100644 index 00000000..0393123c --- /dev/null +++ b/cmd/mesh-controller/composed_rejudge.go @@ -0,0 +1,169 @@ +package main + +import ( + "context" + "errors" + "fmt" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" +) + +// The judgement beside each composition is rewritten when what it is judged against or with changes (novox/hq ADR +// 0539 clauses 2 and 14): when the node's have of the module changes (it runs another composition, or holds), and when +// a person sets the module's policy. Heard as the have bucket's writes and the `upgrade` verb, never on a timer. + +// rejudge writes again the judgement and policy beside every composition of module on the node at, where they differ. +func rejudge(ctx context.Context, at composedPlace, module string) (int, error) { + lister, err := at.bucket.ListKeys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return 0, nil + } + if err != nil { + return 0, err + } + wrote := 0 + var errs []error + for key := range lister.Keys() { + node, m, _, isNode, err := sdkdelivery.ParseComposedKey(key) + if err != nil || isNode || node != at.node || m != module { + continue + } + c, revision, err := readComposed(ctx, at.bucket, key) + if err != nil || revision == 0 { + continue + } + c.Signature = nil + digest, err := sdkdelivery.ComposedDigestOf(c) + if err != nil { + continue + } + judged, err := judgeMove(ctx, at, key, c, digest) + if err != nil { + errs = append(errs, err) + continue + } + w, err := keepDigest(ctx, at.digests, key, judged) + if err != nil { + errs = append(errs, err) + } + if w { + wrote++ + } + } + return wrote, errors.Join(errs...) +} + +// rejudgeOn opens the node's buckets and rejudges module there, under the lease. +func rejudgeOn(ctx context.Context, open *stores, nodes []string, module string) error { + if _, err := theLease.epoch(ctx); err != nil { + return err + } + bus, err := aBus() + if err != nil { + return err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return err + } + upgrades, err := open.inventory.Upgrades(ctx) + if err != nil { + return err + } + var errs []error + for _, node := range nodes { + at, err := openComposedPlace(ctx, js, bus.Conn(), node) + if err != nil { + errs = append(errs, err) + continue + } + at.policy = func(m string) (string, string, string) { + u, ok := upgrades[m] + if !ok { + return "", "", "" + } + return u.Policy(), u.From, u.Why + } + if _, err := rejudge(ctx, at, module); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// rejudgeModule rejudges a module on every node it is assigned to: what a person's policy changes. +func rejudgeModule(ctx context.Context, open *stores, module string) { + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err) + return + } + for _, e := range entries { + if e.Manifest.Module == module && len(e.On) > 0 { + if err := rejudgeOn(ctx, open, e.On, module); err != nil { + fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err) + } + } + } +} + +// watchHavesForJudgement rejudges a node's module each time its have changes, until ctx ends: never on the watch's own +// goroutine. +func watchHavesForJudgement(ctx context.Context, open *stores) { + bus, err := aBus() + if err != nil { + fmt.Printf("compositions: the haves are not watched: %v\n", err) + return + } + js, err := jetstream.New(bus.Conn()) + if err != nil { + bus.Close() + return + } + kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket) + if err != nil { + bus.Close() + fmt.Printf("compositions: the haves are not watched: %v\n", err) + return + } + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + bus.Close() + fmt.Printf("compositions: the haves are not watched: %v\n", err) + return + } + go func() { + defer bus.Close() + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + node, module, isNode, err := sdkdelivery.ParseHaveKey(e.Key()) + if err != nil || isNode { + continue + } + inBackground(func() { + if err := rejudgeOn(ctx, open, []string{node}, module); err != nil { + fmt.Printf("compositions: %s on %s is not judged again: %v\n", module, node, err) + } + // A node's node-engine moved: the declarations' stream goes once every node runs the new one. + if module == nodeEngineModule { + if _, err := retireDeclarations(ctx, open); err != nil { + fmt.Printf("compositions: whether the declarations' stream may go cannot be read: %v\n", err) + } + } + }) + } + } + }() +} diff --git a/cmd/mesh-controller/composed_served_integration_test.go b/cmd/mesh-controller/composed_served_integration_test.go new file mode 100644 index 00000000..56a5abe4 --- /dev/null +++ b/cmd/mesh-controller/composed_served_integration_test.go @@ -0,0 +1,615 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "golang.org/x/crypto/bcrypt" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/chain" + "github.com/novox/mesh-controller/internal/identity" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The compositions through the controller whole (novox/hq ADR 0539; the review's G1): the real store, the controller's +// signing key, and a bus that runs the user list the controller composes from that store, so a subject release 6 uses +// and the list does not grant is refused as it would be on the mesh. serve() runs whole on it; the builder and the +// indexer are their own accounts writing their own keys; a rotation is the operator's verb. +type servedMesh struct { + open *stores + controller jetstream.JetStream + builder jetstream.JetStream + indexer jetstream.JetStream +} + +// keyOfManifest is a done key of a module at fp with no artifacts, its manifest the given one, and the indexer's trunk +// value naming it in the catalogue's repository at commit. +func keyOfManifest(t *testing.T, m catalogue.Manifest, fp, commit string) (raw []byte, k sdkdelivery.BuilderBuild, trunk sdkdelivery.Module) { + t.Helper() + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + digest := sdkdelivery.ManifestDigest(raw) + return raw, sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "anchor", Manifest: digest}, + sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: commit, Path: "modules/" + m.Module} +} + +func aServedMesh(t *testing.T) *servedMesh { + t.Helper() + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + withTrunkList(t, true) + ident := identity.ForTest(t) + if _, err := ident.Establish(ctx); err != nil { + t.Fatal(err) + } + + // The bus, running the user list the controller composes from the store, with the builder and the indexer as the + // accounts their own manifests make (each writes its own state). + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var controller broker.Principal + for _, p := range users { + if p.Kind == broker.KindController { + controller = p + } + } + password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: controller.Username(), Kind: inventory.BusController}) + if err != nil { + t.Fatal(err) + } + builder := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.BuilderModule, State: []string{"builds"}} + indexer := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.IndexerModule, State: []string{"modules", "manifests"}} + kept, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for name, u := range kept { + hashes[name] = u.PasswordHash + } + fixture, _ := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost) + hashes[builder.Username()], hashes[indexer.Username()] = string(fixture), string(fixture) + filled, missing := broker.WithPasswords([]broker.Principal{controller, builder, indexer}, hashes) + if len(missing) > 0 { + t.Fatalf("no password for %v", missing) + } + accounts, err := broker.ComposeAccounts(filled) + if err != nil { + t.Fatal(err) + } + conf := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatalf("the composed user list does not parse: %v", err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(s.Shutdown) + u, _ := url.Parse(s.ClientURL()) + u.User = url.UserPassword(controller.Username(), password) + t.Setenv(broker.NATSVar, u.String()) + + connect := func(user, pw string) jetstream.JetStream { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, pw), nats.CustomInboxPrefix("_INBOX."+user)) + if err != nil { + t.Fatalf("%s cannot connect: %v", user, err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + return js + } + m := &servedMesh{open: open, controller: connect(controller.Username(), password), + builder: connect(builder.Username(), "pw"), indexer: connect(indexer.Username(), "pw")} + + // The test's own inventory on the bus's chain, as the serving controller's will be: what is registered and assigned + // below is in the chain it reads (assignments and builds live on the bus, ADR 0408). + chainConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(controller.Username(), password), + nats.CustomInboxPrefix("_INBOX."+controller.Username())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(chainConn.Close) + // The controller's own buckets, the chain's among them, as it asserts them at its start. + asserter, err := broker.Dial(u.String()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(asserter.Close) + if err := asserter.EnsureControllerBuckets(); err != nil { + t.Fatalf("the controller's buckets: %v", err) + } + c, err := chain.OnTheBus(ctx, chainConn, func(context.Context) error { return nil }) + if err != nil { + t.Fatal(err) + } + inv.UseChain(c) + + // A provider and its consumer (bindings_test.go's store and board), registered from done keys of the catalogue's + // repository, so each has a fingerprint of the new form; the store holds the mesh's seat on anchor, the board on + // laptop is bound to it. + shelf := map[string]catalogue.Manifest{} + for _, m := range storeManifests() { + shelf[m.Module] = m + } + // The board is given its credential in a file, as a consumer of a database is. + board := shelf["board"] + board.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"} + shelf["board"] = board + for i, module := range []string{"store", "board"} { + raw, k, trunk := keyOfManifest(t, shelf[module], fpOf(byte('a'+i)), "c0ffee0123456789") + keys := trunkKeys{keyBuckets: keyBuckets{keys: map[string]builderkeys.Key{module: aBuilderKey(module, trunk.FP2, k)}, + manifests: map[string]string{trunk.Manifest: string(raw)}}, trunk: trunk} + if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil { + t.Fatalf("%s is not registered from its key: %v", module, err) + } + } + // The private network on both machines, as aMesh assigns it, now on the chain the controller reads. + for _, node := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, node, overlay.Name); err != nil { + t.Fatal(err) + } + } + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "store"); err != nil { + t.Fatal(err) + } + if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "laptop", "board"); err != nil { + t.Fatal(err) + } + if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { + t.Fatal(err) + } + + // The builder's and the indexer's buckets as their own parts make them, there before the controller starts (its + // watch of their keys begins at its start); and the plans' buckets a controller of release 5 left. + for _, b := range []string{sdkdelivery.BuilderBuildsBucket, sdkdelivery.ModulesBucket, sdkdelivery.ManifestsBucket} { + if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil { + t.Fatalf("the bucket %s: %v", b, err) + } + } + for _, b := range broker.PlanStateBuckets { + if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + + keepWhatServeSets(t) + serving, stop := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- serve(serving) }() + t.Cleanup(func() { + stop() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Log("the controller did not stop within 30 s") + } + // A stopping controller refuses every verb from then on as a handover (issue 289), which its process exits + // after; this one is the test process, so a later test's verb would be refused without running (issue 1945: + // the conditions verb's own process was never started). Its stop marks it once the serving context ends. + for deadline := time.Now().Add(5 * time.Second); !handingOver.Load() && time.Now().Before(deadline); { + time.Sleep(10 * time.Millisecond) + } + handingOver.Store(false) + }) + // Served once both nodes have their own composition and their module's: the start's pass wrote them all. A node's + // compositions are written in key order, so anchor's own comes before its store's (issue 1945: a test that reads the + // revisions at once must not catch the pass midway). + for _, k := range []struct{ node, module, fp string }{{"anchor", "", ""}, {"laptop", "", ""}, {"anchor", "store", fpOf('a')}, + {"laptop", "board", fpOf('b')}} { + key, _ := sdkdelivery.ComposedNodeKey(k.node) + if k.module != "" { + key, _ = sdkdelivery.ComposedKey(k.node, k.module, k.fp) + } + m.waitFor(t, done, k.node, key, 0) + } + return m +} + +// aBuilderKey is the builder's done key of module at fp, as the take-in reads it. +func aBuilderKey(module, fp string, b sdkdelivery.BuilderBuild) builderkeys.Key { + return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, Build: b} +} + +// waitFor waits until the node's composition bucket holds key at a revision above after, failing when the controller +// stops or a minute passes: a test's wait for what an event made, never a loop of the code under test. +func (m *servedMesh) waitFor(t *testing.T, done chan error, node, key string, after uint64) uint64 { + t.Helper() + bucket, _ := sdkdelivery.ComposedBucketFor(node) + deadline := time.Now().Add(time.Minute) + for time.Now().Before(deadline) { + select { + case err := <-done: + t.Fatalf("the controller stopped: %v", err) + default: + } + if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil { + if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after { + return e.Revision() + } + } + time.Sleep(200 * time.Millisecond) + } + t.Fatalf("%s in %s was not written within a minute; %s", key, bucket, m.whatIsThere(t, node)) + return 0 +} + +// whatIsThere says, for a failure, what the node's bucket holds, the ids its own composition carries and the +// fingerprints the controller has registered. +func (m *servedMesh) whatIsThere(t *testing.T, node string) string { + t.Helper() + bucket, _ := sdkdelivery.ComposedBucketFor(node) + out := "keys:" + kv, err := m.controller.KeyValue(context.Background(), bucket) + if err != nil { + return out + " " + err.Error() + } + lister, err := kv.ListKeys(context.Background()) + if err == nil { + for k := range lister.Keys() { + out += " " + k + } + } + if lister, err := kv.ListKeys(context.Background()); err == nil { + for k := range lister.Keys() { + if e, err := kv.Get(context.Background(), k); err == nil { + if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil { + out += fmt.Sprintf("; %s at revision %d carries", k, e.Revision()) + for _, r := range c.Resources { + var head struct{ ID string } + _ = json.Unmarshal(r, &head) + out += " " + head.ID + } + } + } + } + } + holders, herr := m.open.inventory.HoldersOf(context.Background(), "postgres-database", "") + out += fmt.Sprintf("; holders of postgres-database: %+v (%v)", holders, herr) + own, _ := sdkdelivery.ComposedNodeKey(node) + if e, err := kv.Get(context.Background(), own); err == nil { + if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil { + out += "; its own carries:" + for _, r := range c.Resources { + var head struct{ ID string } + _ = json.Unmarshal(r, &head) + out += " " + head.ID + } + } + } + fps, err := currentFingerprints(context.Background(), m.open.inventory) + out += fmt.Sprintf("; fingerprints registered: %v (%v)", fps, err) + // And what composing the node says now, in this process, under the serving controller's lease. + _, cerr := composeModules(context.Background(), m.open, []string{node}) + return out + fmt.Sprintf("; composing %s now: %v", node, cerr) +} + +// revisions are a node's composition keys and their revisions. +func (m *servedMesh) revisions(t *testing.T, node string) map[string]uint64 { + t.Helper() + bucket, _ := sdkdelivery.ComposedBucketFor(node) + kv, err := m.controller.KeyValue(t.Context(), bucket) + if err != nil { + t.Fatal(err) + } + lister, err := kv.ListKeys(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]uint64{} + for k := range lister.Keys() { + if e, err := kv.Get(t.Context(), k); err == nil { + out[k] = e.Revision() + } + } + return out +} + +// declarationsSent is how many declarations the bus's stream of them holds: none is ever sent in release 6. +func (m *servedMesh) declarationsSent(t *testing.T) uint64 { + t.Helper() + s, err := m.controller.Stream(t.Context(), broker.DeclarationsStream) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return 0 + } + if err != nil { + t.Fatal(err) + } + info, err := s.Info(t.Context()) + if err != nil { + t.Fatal(err) + } + return info.State.Msgs +} + +// 1945/AC6, AC14: started whole on a bus running its own user list, the controller composes every node and every +// assigned module, deletes the plans' buckets and says each as bucket-deleted, and sends no declaration. +func TestServedTheStartComposesEveryNodeAndDeletesThePlansBuckets(t *testing.T) { + m := aServedMesh(t) + for _, k := range []struct{ node, module string }{{"anchor", "store"}, {"laptop", "board"}} { + fp := map[string]string{"store": fpOf('a'), "board": fpOf('b')}[k.module] + key, _ := sdkdelivery.ComposedKey(k.node, k.module, fp) + m.waitFor(t, nil, k.node, key, 0) + } + for _, b := range broker.PlanStateBuckets { + if _, err := m.controller.KeyValue(t.Context(), b); !errors.Is(err, jetstream.ErrBucketNotFound) { + t.Errorf("%s is still there: %v", b, err) + } + } + if n := m.declarationsSent(t); n != 0 { + t.Errorf("%d declarations were sent", n) + } +} + +// 1945/AC3: the builder writes the done key of a new fingerprint of an assigned module and the indexer its trunk +// value: the controller, hearing the keys, registers it, writes its composition on the node it is assigned to and +// `grants..`; nothing is sent. +func TestServedADoneKeyIsComposedAndGranted(t *testing.T) { + m := aServedMesh(t) + ctx := t.Context() + shelf := map[string]catalogue.Manifest{} + for _, mf := range storeManifests() { + shelf[mf.Module] = mf + } + next := shelf["board"] + next.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"} + next.Version = "2" + fp := fpOf('c') + raw, k, trunk := keyOfManifest(t, next, fp, "d00d0123456789ab") + manifests, err := m.indexer.KeyValue(ctx, sdkdelivery.ManifestsBucket) + if err != nil { + t.Fatal(err) + } + value, _ := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "board", JSON: string(raw)}) + if _, err := manifests.Put(ctx, trunk.Manifest, value); err != nil { + t.Fatalf("the indexer cannot write its manifest: %v", err) + } + builds, err := m.builder.KeyValue(ctx, sdkdelivery.BuilderBuildsBucket) + if err != nil { + t.Fatal(err) + } + key, _ := sdkdelivery.BuilderBuildKey("board", fp) + value, _ = sdkdelivery.EncodeBuilderBuild(k) + if _, err := builds.Put(ctx, key, value); err != nil { + t.Fatalf("the builder cannot write its key: %v", err) + } + modules, err := m.indexer.KeyValue(ctx, sdkdelivery.ModulesBucket) + if err != nil { + t.Fatal(err) + } + mk, _ := sdkdelivery.ModulesKey("board") + value, _ = sdkdelivery.EncodeModule(trunk) + if _, err := modules.Put(ctx, mk, value); err != nil { + t.Fatalf("the indexer cannot write its trunk value: %v", err) + } + composed, _ := sdkdelivery.ComposedKey("laptop", "board", fp) + m.waitFor(t, nil, "laptop", composed, 0) + grants, err := m.controller.KeyValue(ctx, broker.GrantsBucket) + if err != nil { + t.Fatal(err) + } + gk, _ := sdkdelivery.ControllerGrantKey("board", fp) + // The take-in writes the grant after the compositions (composed_keys.go), so it is waited for, not read in the instant + // the composition shows: a loaded build node read it a moment early (issue 1945). + becomes(t, 10*time.Second, "grants."+gk+" written", func() bool { + _, err := grants.Get(ctx, gk) + return err == nil + }) + if n := m.declarationsSent(t); n != 0 { + t.Errorf("%d declarations were sent", n) + } +} + +// 1945/AC2: the operator rotates the store's credential, the real verb: the compositions that carry it (the store's +// grants on anchor, and the board's on laptop when it carries the credential) get a new revision, every other +// composition keeps its own, and nothing is sent. +func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) { + m := aServedMesh(t) + // Everything that writes or discards a credential row, and what composing each node saw and decided, in order, with the + // callers: one failure shows the cause (issue 1945). + var traceMu sync.Mutex + var trace []string + note := func(line string) { + traceMu.Lock() + defer traceMu.Unlock() + trace = append(trace, time.Now().Format("15:04:05.000")+" "+line) + } + inventory.SecretTrace = func(op, what string) { note("secret " + op + ": " + what) } + composeTrace = func(line string) { note("compose " + line) } + t.Cleanup(func() { inventory.SecretTrace, composeTrace = nil, nil }) + traced := func() string { + traceMu.Lock() + defer traceMu.Unlock() + return strings.Join(trace, "\n ") + } + defer func() { + if t.Failed() { + t.Logf("TRACE\n %s", traced()) + } + }() + before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")} + holders, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "") + var waited uint64 + // The verb runs here as its child process runs it: its inventory takes the chain from the bus the served controller keeps it + // on (the registrations and assignments are there), not the one in memory the fixture's store was made with. + inventory.OnTheBusChainForTest(t) + said, err := capturedOutput(func() error { + if err := rotateCommand(t.Context(), []string{"postgres-database"}); err != nil { + return err + } + // The serving controller's own lines while the rotation lands are captured too. + boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b')) + waited = m.waitForOrSay(t, "laptop", boardKey, before["laptop"][boardKey]) + return nil + }) + if err != nil { + t.Fatalf("rotate: %v\n%s", err, said) + } + t.Logf("rotate said (holders before: %+v):\n%s", holders, said) + defer func() { + if t.Failed() { + t.Logf("rotate said:\n%s", said) + } + }() + boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b')) + storeKey, _ := sdkdelivery.ComposedKey("anchor", "store", fpOf('a')) + // The board's sealed credential file is new: its composition moves with the rotation. + if rev := waited; rev == 0 { + var kept []string + for _, line := range strings.Split(said, "\n") { + for _, w := range []string{"board", "postgres", "secret", "credential", "laptop", "withdr", "retir", "error", "refused"} { + if strings.Contains(line, w) { + kept = append(kept, line) + break + } + } + } + said = strings.Join(kept, " | ") + after, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "") + in, ierr := inputFor(t.Context(), m.open) + var credentials string + if ierr == nil { + credentials = fmt.Sprintf("would make %+v; credentials %+v", in.wouldMake, in.Credentials) + } + state, serr := m.open.inventory.PairCredentialState(t.Context(), inventory.PairCredential{Name: "postgres-database", + Consumer: "laptop", ConsumerModule: "board", Provider: "anchor"}) + d, derr := composeSend(t.Context(), m.open, "laptop") + var sealed []string + for _, r := range d.Resources { + if r["id"] == "board.secret-postgres-database" { + sealed = append(sealed, fmt.Sprint(r["sealed"])[:min(24, len(fmt.Sprint(r["sealed"])))]) + } + } + diag := fmt.Sprintf("state %+v %v; compose now %v sealed %v; %s %v", state, serr, derr, sealed, credentials, ierr) + // One line: the verdict keeps only a failure's first line (issue 1921). + t.Fatalf("the board's composition did not move within a minute after the rotation (revision before %d); TRACE %s; LINES %s; "+ + "holders before %+v, after %+v; DIAG %s; %s", before["laptop"][boardKey], + strings.Join(strings.Fields(strings.ReplaceAll(traced(), "\n", " | ")), " "), strings.Join(strings.Fields(said), " "), + holders, after, diag, strings.Join(strings.Fields(m.whatIsThere(t, "laptop")), " ")) + } + moved := map[string]bool{} + for node, was := range before { + for k, rev := range m.revisions(t, node) { + if rev != was[k] { + moved[k] = true + } + } + } + for k := range moved { + if k != boardKey && k != storeKey { + t.Errorf("%s moved: only the compositions carrying the credential do", k) + } + } + if !moved[boardKey] { + t.Errorf("the board's composition did not move: %v", moved) + } + if n := m.declarationsSent(t); n != 0 { + t.Errorf("%d declarations were sent", n) + } +} + +// waitForOrSay is waitFor that answers 0 instead of failing, for a caller that says more. +func (m *servedMesh) waitForOrSay(t *testing.T, node, key string, after uint64) uint64 { + t.Helper() + bucket, _ := sdkdelivery.ComposedBucketFor(node) + deadline := time.Now().Add(time.Minute) + for time.Now().Before(deadline) { + if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil { + if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after { + return e.Revision() + } + } + time.Sleep(200 * time.Millisecond) + } + return 0 +} + +// capturedOutput runs act with this process's standard output captured, and answers what it printed. +func capturedOutput(act func() error) (string, error) { + r, w, err := os.Pipe() + if err != nil { + return "", act() + } + was := os.Stdout + os.Stdout = w + done := make(chan string) + go func() { + var b strings.Builder + buf := make([]byte, 4096) + for { + n, err := r.Read(buf) + b.Write(buf[:n]) + if err != nil { + break + } + } + done <- b.String() + }() + actErr := act() + os.Stdout = was + w.Close() + return <-done, actErr +} + +// keepWhatServeSets gives back, once the served controller stopped, what serve sets for the rest of its process, which +// a controller's process exits after and the test process does not (issue 1945): the served mark in the environment, the +// hand-act log's connection, the keeper, the status and the event publishers. Registered before the controller starts, so +// it runs after the cleanup that stops it. +func keepWhatServeSets(t *testing.T) { + t.Helper() + t.Setenv(servedVar, "") + unheld, status, given, check, follow, keeper, hand := logUnheldChanges, statusFrom, givenEvents, checkEvents, + link.Calls.Follow, conditionsFrom, handActConn + t.Cleanup(func() { + logUnheldChanges, statusFrom, givenEvents, checkEvents, link.Calls.Follow, conditionsFrom, handActConn = + unheld, status, given, check, follow, keeper, hand + }) +} diff --git a/cmd/mesh-controller/composed_start.go b/cmd/mesh-controller/composed_start.go new file mode 100644 index 00000000..6b5b0f61 --- /dev/null +++ b/cmd/mesh-controller/composed_start.go @@ -0,0 +1,341 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// What the controller does at its start under the lease (novox/hq ADR 0539 clauses 10 and 13): +// +// - it deletes the buckets of the plans' state this release deleted, each said once in its log and as its event +// `bucket-deleted`: a later start finds none and says nothing; +// - it composes every node, so every assigned module has its composition at the fingerprint it is registered at +// and each node its own (ADR 0539 clause 11: the new controller writes every composition at its start); +// - and it composes, on the control-node, the controller it replaced at that controller's own fingerprint, from that +// build's manifest, so the operator's way back is one key at the terminal (clause 10). + +// planStateBucketDeleter is what deleting the plans' buckets needs of the bus: jetstream.JetStream, or a test's. +type planStateBucketDeleter interface { + DeleteKeyValue(ctx context.Context, bucket string) error +} + +// deletePlanStateBuckets deletes each bucket of the plans' state the bus still holds and says each it deleted; it +// answers those it deleted. A bucket not there is no error and is not said. +func deletePlanStateBuckets(ctx context.Context, js planStateBucketDeleter, say func(bucket string)) ([]string, error) { + var deleted []string + var errs []error + for _, bucket := range broker.PlanStateBuckets { + err := js.DeleteKeyValue(ctx, bucket) + switch { + case errors.Is(err, jetstream.ErrBucketNotFound), errors.Is(err, jetstream.ErrStreamNotFound): + continue + case err != nil: + errs = append(errs, fmt.Errorf("deleting %s: %w", bucket, err)) + continue + } + deleted = append(deleted, bucket) + say(bucket) + } + return deleted, errors.Join(errs...) +} + +// BucketDeleted is the body of the controller's event `bucket-deleted`. +type BucketDeleted struct { + Bucket string `json:"bucket"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// sayBucketDeleted says a deleted bucket in the log and as the controller's event. +func sayBucketDeleted(ctx context.Context, bucket string) { + sayDeleted(ctx, bucket, "a bucket of the plans' state, which release 6 deleted (novox/hq ADR 0539 clause 13)") +} + +// sayDeleted says a deleted bucket or stream, with why, in the log and as the controller's event bucket-deleted. +func sayDeleted(ctx context.Context, bucket, why string) { + fmt.Printf("start: deleted %s: %s\n", bucket, why) + bus := checkEvents + if bus == nil { + return + } + body, err := json.Marshal(BucketDeleted{Bucket: bucket, Why: why, At: time.Now().UTC()}) + if err != nil { + return + } + saying, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer stop() + if err := bus.PublishSeatEvent(saying, link.MeshControllerSeat, "bucket-deleted", body); err != nil { + fmt.Printf("start: that %s was deleted could not be said as the event bucket-deleted: %v\n", bucket, err) + } +} + +// composeAtStart is the start's pass: the plans' buckets deleted, every node composed, the replaced controller +// composed. Said, never fatal: what is not composed now is composed by the next act that touches it. +func composeAtStart(ctx context.Context, open *stores) { + if bus, err := aBus(); err != nil { + fmt.Printf("start: the plans' buckets are not looked at: %v\n", err) + } else { + if js, err := jetstream.New(bus.Conn()); err == nil { + if _, err := deletePlanStateBuckets(ctx, js, func(b string) { sayBucketDeleted(ctx, b) }); err != nil { + fmt.Printf("start: %v\n", err) + } + } + bus.Close() + } + nodes, err := open.inventory.Nodes(ctx) + if err != nil { + fmt.Printf("start: nothing is composed: %v\n", err) + return + } + names := make([]string, 0, len(nodes)) + for _, n := range nodes { + names = append(names, n.Name) + } + sort.Strings(names) + if _, err := composeModules(ctx, open, names); err != nil { + fmt.Printf("start: %v\n", err) + } + if err := composeReplacedController(ctx, open); err != nil { + fmt.Printf("start: the controller this one replaced is not composed: %v\n", err) + } + if _, err := retireDeclarations(ctx, open); err != nil { + fmt.Printf("start: whether the declarations' stream may go cannot be read: %v\n", err) + } +} + +// nodeEngineModule is the node-engine's module. +const nodeEngineModule = "mesh-host" + +// declarationsDone says every node runs the node-engine at fingerprint current, as its have names: no node still reads +// a declaration, and none away will need its last one (ADR 0539 clause 11). No node, or no fingerprint of the new +// form, is not done. +func declarationsDone(nodes []string, runs map[string]string, current string) bool { + if len(nodes) == 0 || !strings.HasPrefix(current, sdkdelivery.SourcePrefix2) { + return false + } + for _, n := range nodes { + if runs[n] != current { + return false + } + } + return true +} + +// declarationsRetiredOn reads whether the declarations are done on the mesh now: every node's have of the node-engine +// against the fingerprint it is registered at. +func declarationsRetiredOn(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream) (bool, error) { + fps, err := currentFingerprints(ctx, inv) + if err != nil { + return false, err + } + nodes, err := inv.Nodes(ctx) + if err != nil { + return false, err + } + names := make([]string, 0, len(nodes)) + runs := map[string]string{} + for _, n := range nodes { + names = append(names, n.Name) + h, found, err := haveOn(ctx, js, n.Name, nodeEngineModule) + if err != nil { + return false, err + } + if found { + runs[n.Name] = h.FP + } + } + return declarationsDone(names, runs, fps[nodeEngineModule]), nil +} + +// retireDeclarations deletes the declarations' stream, with every node's consumer on it, once every node runs the new +// node-engine, and says it once; a stream already gone is said by nobody. It answers whether the declarations are done. +func retireDeclarations(ctx context.Context, open *stores) (bool, error) { + if _, err := theLease.epoch(ctx); err != nil { + return false, err + } + bus, err := aBus() + if err != nil { + return false, err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return false, err + } + done, err := declarationsRetiredOn(ctx, open.inventory, js) + if err != nil || !done { + return done, err + } + err = js.DeleteStream(ctx, broker.DeclarationsStream) + switch { + case errors.Is(err, jetstream.ErrStreamNotFound): + return true, nil + case err != nil: + return true, fmt.Errorf("deleting the stream %s: %w", broker.DeclarationsStream, err) + } + sayDeleted(ctx, broker.DeclarationsStream, "the stream of each node's last declaration: every node runs the new "+ + "node-engine, which reads none (novox/hq ADR 0539 clause 11)") + return true, nil +} + +// replacedController is the controller a control-node ran before this one: its fingerprint and manifest. +type replacedController struct { + node, fp string + manifest catalogue.Manifest +} + +// replacedControllers are, for each node the controller is assigned to, the controller it replaced: the newest build of +// the controller that worked, whose fingerprint is of the new form and not the one the controller is registered at now +// (the assignments name no build since ADR 0539 §12, so the build records say it). None when there is no such build. +func replacedControllers(ctx context.Context, open *stores) ([]replacedController, error) { + inv := open.inventory + fps, err := currentFingerprints(ctx, inv) + if err != nil { + return nil, err + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, err + } + var on []string + for _, e := range entries { + if e.Manifest.Module == controllerModule { + on = e.On + } + } + if len(on) == 0 { + return nil, nil + } + builds, err := inv.Builds(ctx, controllerModule, 20) + if err != nil { + return nil, err + } + before, found := replacedBuild(builds, fps[controllerModule]) + if !found { + return nil, nil + } + var m catalogue.Manifest + if err := json.Unmarshal(before.Manifest, &m); err != nil { + return nil, fmt.Errorf("the manifest of %s's build %s: %w", controllerModule, before.ID, err) + } + var out []replacedController + for _, node := range on { + out = append(out, replacedController{node: node, fp: before.SourceFingerprint, manifest: m}) + } + return out, nil +} + +// replacedBuild is, of a module's builds newest first, the newest that worked, kept its manifest, and names a +// fingerprint of the new form other than current. +func replacedBuild(builds []inventory.Build, current string) (inventory.Build, bool) { + for _, b := range builds { + if b.Worked() && b.SourceFingerprint != current && strings.HasPrefix(b.SourceFingerprint, sdkdelivery.SourcePrefix2) && + len(b.Manifest) > 0 { + return b, true + } + } + return inventory.Build{}, false +} + +// composeReplacedController writes, on each control-node, the controller's composition at the fingerprint of the one +// it replaced: the node composed with that build's manifest in place of the controller's registered one, in the +// reading mode (nothing made), and only the controller's composition of it kept. +func composeReplacedController(ctx context.Context, open *stores) error { + if _, err := theLease.epoch(ctx); err != nil { + return err + } + replaced, err := replacedControllers(ctx, open) + if err != nil || len(replaced) == 0 { + return err + } + bus, err := aBus() + if err != nil { + return err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return err + } + sign, err := controllerSigner(ctx, open) + if err != nil { + return err + } + var errs []error + for _, r := range replaced { + in, err := inputFor(ctx, open) + if err != nil { + return err + } + c, err := replacedComposition(in.Input, r) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", r.node, err)) + continue + } + at, err := openComposedPlace(ctx, js, bus.Conn(), r.node) + if err != nil { + errs = append(errs, err) + continue + } + at.sign = sign + key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp) + if wrote, err := writeComposed(ctx, at, key, c); err != nil { + errs = append(errs, err) + } else if wrote { + fmt.Printf("start: composed %s, the controller this one replaced, on %s: the way back is want.%s.%s naming it\n", + short(r.fp), r.node, r.node, controllerModule) + } + } + return errors.Join(errs...) +} + +// replacedComposition is the controller's composition on r's node at r's fingerprint: in composed with r's manifest +// in place of the controller's registered one. +func replacedComposition(in compose.Input, r replacedController) (sdkdelivery.Composed, error) { + shelf := make(map[string]catalogue.Manifest, len(in.Catalogue)) + for k, v := range in.Catalogue { + shelf[k] = v + } + shelf[controllerModule] = r.manifest + in.Catalogue = shelf + all, _, err := compose.Compositions(in, r.node, map[string]string{controllerModule: r.fp}, compose.Reading) + if err != nil { + return sdkdelivery.Composed{}, err + } + key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp) + c, ok := all[key] + if !ok { + return sdkdelivery.Composed{}, fmt.Errorf("%s composes no %s with the replaced build's manifest", r.node, controllerModule) + } + return c, nil +} + +// declarationsRetiredNow says the declarations are done on the mesh now, for a raise of the bus's objects; a bus or a +// store it cannot read is not done, so the stream is asserted as before. +func declarationsRetiredNow(ctx context.Context, inv *inventory.Inventory) bool { + bus, err := aBus() + if err != nil { + return false + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return false + } + done, err := declarationsRetiredOn(ctx, inv, js) + return err == nil && done +} diff --git a/cmd/mesh-controller/composed_start_test.go b/cmd/mesh-controller/composed_start_test.go new file mode 100644 index 00000000..35831c35 --- /dev/null +++ b/cmd/mesh-controller/composed_start_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// fakeBuckets is the bus's buckets by name, for deleting. +type fakeBuckets map[string]bool + +func (f fakeBuckets) DeleteKeyValue(_ context.Context, bucket string) error { + if !f[bucket] { + return jetstream.ErrBucketNotFound + } + delete(f, bucket) + return nil +} + +// 1945/AC14 (unit half): the first start deletes the merge window's, the silent sends' and the building plans' buckets +// and says each once; a later start finds none and says nothing; a bucket of anything else is left alone. +func TestTheFirstStartDeletesThePlansBucketsAndSaysEachOnce(t *testing.T) { + bus := fakeBuckets{"mesh-controller_merge-window": true, "mesh-controller_rollout-silent": true, + "mesh-controller_plans-building": true, "mesh-controller_grants": true} + var said []string + deleted, err := deletePlanStateBuckets(context.Background(), bus, func(b string) { said = append(said, b) }) + if err != nil || len(deleted) != 3 || strings.Join(said, " ") != strings.Join(broker.PlanStateBuckets, " ") { + t.Fatalf("first start: deleted %v, said %v, %v", deleted, said, err) + } + if !bus["mesh-controller_grants"] || len(bus) != 1 { + t.Errorf("the bus holds %v after the first start", bus) + } + said = nil + if deleted, err := deletePlanStateBuckets(context.Background(), bus, func(b string) { said = append(said, b) }); err != nil || len(deleted)+len(said) != 0 { + t.Errorf("a later start deleted %v and said %v (%v)", deleted, said, err) + } +} + +// 1945/AC6 (unit half): the controller this one replaced is composed on its node from its own build's manifest at its +// own fingerprint: the composition holds what that manifest declares, not what the controller registered now declares. +func TestTheReplacedControllerIsComposedFromItsOwnManifest(t *testing.T) { + now := catalogue.Manifest{Module: controllerModule, Version: "6", Resources: []map[string]any{ + {"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "six\n"}}} + before := catalogue.Manifest{Module: controllerModule, Version: "5", Resources: []map[string]any{ + {"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "five\n"}}} + in := compose.Input{Catalogue: map[string]catalogue.Manifest{controllerModule: now}, + Nodes: []compose.Node{{Name: "anchor", Assigned: []string{controllerModule}, Stated: true}}} + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("5", 64) + c, err := replacedComposition(in, replacedController{node: "anchor", fp: fp, manifest: before}) + if err != nil { + t.Fatal(err) + } + if c.Module != controllerModule || c.FP != fp || c.Node != "anchor" || len(c.Resources) != 1 { + t.Fatalf("composed %s at %s on %s with %d resources", c.Module, c.FP, c.Node, len(c.Resources)) + } + var f struct{ Content string } + _ = json.Unmarshal(c.Resources[0], &f) + if f.Content != "five\n" { + t.Errorf("the replaced controller's composition holds %q, its own manifest says five", f.Content) + } + if in.Catalogue[controllerModule].Version != "6" { + t.Errorf("composing the replaced controller changed the Input's catalogue") + } +} + +// 1945/AC6 (unit half): the controller replaced is the newest earlier build of the controller that worked and names a +// fingerprint of the new form, not the one registered now. +func TestTheReplacedControllerIsItsNewestEarlierBuild(t *testing.T) { + now, before, older := sdkdelivery.SourcePrefix2+strings.Repeat("6", 64), sdkdelivery.SourcePrefix2+strings.Repeat("5", 64), + sdkdelivery.SourcePrefix2+strings.Repeat("4", 64) + m := []byte(`{"module":"mesh-controller","version":"5"}`) + builds := []inventory.Build{ + {ID: "c6", SourceFingerprint: now, Manifest: m}, + {ID: "c5-failed", SourceFingerprint: before, Failed: "no", Manifest: m}, + {ID: "c5", SourceFingerprint: before, Manifest: m}, + {ID: "c4", SourceFingerprint: older, Manifest: m}, + } + if b, ok := replacedBuild(builds, now); !ok || b.ID != "c5" { + t.Errorf("the replaced build is %+v (%v), want c5", b, ok) + } + if _, ok := replacedBuild(builds[:1], now); ok { + t.Errorf("a controller with no earlier build replaced one") + } +} + +// 1945/AC14 (unit half): the declarations' stream stays while one node's have names another node-engine than the one +// registered, or none, and goes once every node's names it. +func TestTheDeclarationsGoOnlyOnceEveryNodeRunsTheNewEngine(t *testing.T) { + n6 := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64) + nodes := []string{"ace", "g14", "novox"} + if declarationsDone(nodes, map[string]string{"ace": n6, "novox": n6}, n6) { + t.Errorf("done while g14 says nothing (it sleeps)") + } + if declarationsDone(nodes, map[string]string{"ace": n6, "g14": "src2:old", "novox": n6}, n6) { + t.Errorf("done while g14 runs the old node-engine") + } + if !declarationsDone(nodes, map[string]string{"ace": n6, "g14": n6, "novox": n6}, n6) { + t.Errorf("not done with every node on the new node-engine") + } + if declarationsDone(nodes, map[string]string{"ace": "", "g14": "", "novox": ""}, "") || declarationsDone(nil, nil, n6) { + t.Errorf("done with no fingerprint registered, or no node") + } +} diff --git a/cmd/mesh-controller/composed_trace.go b/cmd/mesh-controller/composed_trace.go new file mode 100644 index 00000000..9ab7782a --- /dev/null +++ b/cmd/mesh-controller/composed_trace.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "fmt" + "sort" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// composeTrace, when set, is told what composing a node saw and decided: the Input's credential state before it composes, and for +// each module the digest it composed against the digest already written. A test sets it to read why a composition did or did not +// move; the controller never does. +var composeTrace func(string) + +// traceInput says what the node's composition will read of the pair credentials: those no row holds (would-make) and those held. +func traceInput(ctx context.Context, open *stores, node string) { + in, err := inputFor(ctx, open) + if err != nil { + composeTrace(fmt.Sprintf("%s: the Input cannot be read: %v", node, err)) + return + } + composeTrace(fmt.Sprintf("%s: before composing, would make %+v; credentials held %+v", node, in.wouldMake, in.Credentials)) +} + +// traceComposed says, per composition of a node, the digest composed now against the one written. +func traceComposed(ctx context.Context, at composedPlace, node string, comps map[string]sdkdelivery.Composed) { + keys := make([]string, 0, len(comps)) + for k := range comps { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + now, err := sdkdelivery.ComposedDigestOf(comps[key]) + if err != nil { + composeTrace(fmt.Sprintf("%s: %s: no digest: %v", node, key, err)) + continue + } + written := "none" + if e, err := at.digests.Get(ctx, key); err == nil { + if v, err := sdkdelivery.DecodeComposedDigest(e.Value()); err == nil { + written = v.Digest + } + } + same := "CHANGED" + if written == now { + same = "same" + } + composeTrace(fmt.Sprintf("%s: %s: composed %.19s, written %.19s: %s", node, key, now, written, same)) + } +} diff --git a/cmd/mesh-controller/composed_userlist_test.go b/cmd/mesh-controller/composed_userlist_test.go new file mode 100644 index 00000000..e79182c7 --- /dev/null +++ b/cmd/mesh-controller/composed_userlist_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// natsComposedWith is the bus module's composition on novox with the given user list: the catalogue's nats manifest as +// captured (testdata/beside/mesh-catalog/modules/nats/module.json.captured), its archives, which only a registration +// turns into references, left out, and its container's built image given as a reference. +func natsComposedWith(t *testing.T, users string) sdkdelivery.Composed { + t.Helper() + raw, err := os.ReadFile("../../testdata/beside/mesh-catalog/modules/nats/module.json.captured") + if err != nil { + t.Fatal(err) + } + var m catalogue.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + var kept []map[string]any + for _, r := range m.Resources { + if _, templated := r["template"]; templated { + continue // a template is rendered from the build's templates archive, which only a registration fetches + } + switch r["type"] { + case "archive": + continue + case "container": + // Its image is named by a build's artifact, which only a registration records: given here as a reference. + delete(r, "artifact") + r["image"] = "nats@sha256:" + strings.Repeat("1", 64) + } + kept = append(kept, r) + } + m.Resources = kept + key := base64.StdEncoding.EncodeToString(make([]byte, 32)) + in := compose.Input{Catalogue: map[string]catalogue.Manifest{m.Module: m}, BusUsers: users, + OwnSecrets: []compose.OwnSecretRow{{Node: "novox", Module: m.Module, Name: "broker", Origin: "made", SealedTo: key, + Sealed: "SEALED-BY-THE-TEST"}}, + Nodes: []compose.Node{{Name: "novox", Assigned: []string{m.Module}, Stated: true, SealingKey: key, + Capabilities: map[string]bool{"container-runtime": true, "systemd": true}}}} + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("e", 64) + all, _, err := compose.Compositions(in, "novox", map[string]string{m.Module: fp}, compose.Foreseeing) + if err != nil { + t.Fatal(err) + } + k, _ := sdkdelivery.ComposedKey("novox", m.Module, fp) + c, ok := all[k] + if !ok { + t.Fatalf("no composition of %s: %v", m.Module, all) + } + return c +} + +// 1945/AC12 and the review's A2 (ADR 0539 clause 7): a revision of the bus module's composition that changes only its +// user list, a file in a directory its container mounts, is judged neither and said to change only the user list, so +// a new credential never waits for a person's yes; a revision that changes anything else beside it is judged as any. +func TestARevisionOfOnlyTheUserListMovesNothing(t *testing.T) { + at, _, _, haves := aPlace(t, "novox") + ctx := context.Background() + before := natsComposedWith(t, "authorization { users = [ {user: a} ] }\n") + after := natsComposedWith(t, "authorization { users = [ {user: a}, {user: b} ] }\n") + if d1, _ := sdkdelivery.ComposedDigestOf(before); d1 == func() string { d, _ := sdkdelivery.ComposedDigestOf(after); return d }() { + t.Fatal("the two user lists composed the same") + } + key, _ := sdkdelivery.ComposedKey("novox", before.Module, before.FP) + if _, err := writeComposed(ctx, at, key, before); err != nil { + t.Fatal(err) + } + runs, _ := sdkdelivery.ComposedDigestOf(before) + haves[before.Module] = sdkdelivery.Have{FP: before.FP, Composed: runs, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + digest, _ := sdkdelivery.ComposedDigestOf(after) + v, err := judgeMove(ctx, at, key, after, digest) + if err != nil { + t.Fatal(err) + } + if v.Judgement != sdkdelivery.JudgedNeither || !v.UserListOnly || v.Against != runs { + t.Errorf("a user-list-only revision is judged %+v", v) + } + // The user list and a container's image beside it: judged as any move, and not user-list-only. + other := after + other.Resources = append([]json.RawMessage{}, after.Resources...) + for i, r := range other.Resources { + var head map[string]any + _ = json.Unmarshal(r, &head) + if head["type"] == "container" { + head["image"] = "nats@sha256:" + strings.Repeat("9", 64) + other.Resources[i], _ = json.Marshal(head) + } + } + d2, _ := sdkdelivery.ComposedDigestOf(other) + v, err = judgeMove(ctx, at, key, other, d2) + if err != nil { + t.Fatal(err) + } + if v.UserListOnly || v.Judgement != sdkdelivery.JudgedRecreates { + t.Errorf("a new image beside the user list is judged %+v", v) + } +} diff --git a/cmd/mesh-controller/composed_writer.go b/cmd/mesh-controller/composed_writer.go new file mode 100644 index 00000000..2529e613 --- /dev/null +++ b/cmd/mesh-controller/composed_writer.go @@ -0,0 +1,370 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" +) + +// The compositions (novox/hq ADR 0539 clauses 1 to 5). Every act that sent a declaration writes compositions instead: +// the controller composes a node as it composed a send (composeSend: the same composition, the same makes), splits the +// declaration by module into one composition per module at the fingerprint the module is registered at, and the node's +// own resources, signs each with its key, and writes those that differ, by compare-and-set while it holds the lease: +// the composition into the node's bucket `composed-`, which only that node reads, and its digest with the +// judgement of the move (composed_judge.go) into `mesh-controller_composed`, which the assigner reads. A file over 64 +// KiB goes by digest into the node's object bucket first. Nothing is published on a node's declaration subject. +// +// **A module with no fingerprint of the new form goes with the node's own composition.** The mesh composes resources +// of modules it provides itself, built by nobody (the private network's mesh-wireguard), and a module registered +// before fingerprints of the new form has none either: no want can ever name such a module (the assigner's first rule +// asks for its done build), so its resources ride in `.node`, which the node's own want names, their `module` +// taken off (a composition holds only its own module's resources); their ids still name the module. +// +// **Derived state, pruned** (clause 4): a module's compositions on a node are kept for its five newest fingerprints and +// for any a want or the node's have names; the rest are deleted with their digests. + +// composedBucketsKV is what the writer needs of a key-value bucket: jetstream.KeyValue, or a test's. +type composedBucketsKV interface { + Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) + Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) + Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) + Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error + ListKeys(ctx context.Context, opts ...jetstream.WatchOpt) (jetstream.KeyLister, error) +} + +// composedPlace is where the writer writes a node's compositions and reads what decides them: the node's bucket, the +// digests, the node's object bucket, the wants and the haves. +type composedPlace struct { + node string + bucket composedBucketsKV + digests composedBucketsKV + objects objectStore + // wanted is the fingerprint the node's want names for a module, "" when none; ran the composition digest its have + // names (the move's judgement reads it) and the fingerprint it runs. + wanted func(ctx context.Context, module string) (string, error) + ran func(ctx context.Context, module string) (sdkdelivery.Have, bool, error) + // lastApplied is the module's slice of the declaration the node last applied, from the kept stream of declarations + // (ADR 0540 clause 9), and that declaration's digest; false when there is none to read. Nil judges without it. + lastApplied func(ctx context.Context, module string) (slice []json.RawMessage, digest string, found bool, err error) + // policy is a module's upgrade policy, where it comes from and why (catalogue.UpgradeOf), carried beside the + // judgement for the assigner's hold (ADR 0539 clause 14). + policy func(module string) (policy, from, why string) + // sign signs a composition with the controller's key. + sign func(c sdkdelivery.Composed) (sdkdelivery.Composed, error) +} + +// compositionsOf splits a node's declaration body, as a send composed it, into its compositions: each module's at the +// fingerprint fps gives it, the node's own with its adoption and provisioner, and every resource of a module fps names +// no fingerprint for in the node's own, its module taken off it and named in moved. Data says the module's manifest +// declares data. +func compositionsOf(node string, body []byte, fps map[string]string, manifests map[string]catalogue.Manifest, runs []string) ( + out map[string]sdkdelivery.Composed, moved []string, err error) { + + var envelope struct { + Resources []map[string]json.RawMessage `json:"resources"` + Adoption *sdkdecl.Adoption `json:"adoption"` + Filled []string `json:"filled_files"` + Provisioner string `json:"provisioner_account"` + } + if err := json.Unmarshal(body, &envelope); err != nil { + return nil, nil, fmt.Errorf("%s's declaration: %w", node, err) + } + inModule := map[string]bool{} + resources := make([]json.RawMessage, 0, len(envelope.Resources)) + for _, r := range envelope.Resources { + var module string + if raw, ok := r["module"]; ok { + _ = json.Unmarshal(raw, &module) + } + if module != "" && !strings.HasPrefix(fps[module], sdkdelivery.SourcePrefix2) { + delete(r, "module") + if !inModule[module] { + inModule[module] = true + moved = append(moved, module) + } + } + raw, err := json.Marshal(r) + if err != nil { + return nil, nil, err + } + resources = append(resources, raw) + } + // A filled file of a module moved into the node's own composition is no module's file any more: the provisioner + // fills it all the same, but a composition carries only its own module's filled paths. + var filled []string + for _, p := range envelope.Filled { + if !filledByMoved(envelope.Resources, p, inModule) { + filled = append(filled, p) + } + } + sort.Strings(moved) + out, err = sdkdelivery.SplitComposed(node, resources, fps, filled, + sdkdelivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner}) + if err != nil { + return nil, nil, err + } + // A module the node runs that composes no resource of its own (all its needs met by others) still has its + // composition, empty, at its fingerprint: its want names that composition (the assigner's seventh rule), and without + // one no want could ever be written. + for _, module := range runs { + fp := fps[module] + if !strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) { + continue + } + key, err := sdkdelivery.ComposedKey(node, module, fp) + if err != nil { + continue + } + if _, ok := out[key]; !ok { + out[key] = sdkdelivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{}} + } + } + for key, c := range out { + if m, ok := manifests[c.Module]; ok && c.Module != "" { + c.Data = len(m.DataItems()) > 0 + out[key] = c + } + } + return out, moved, nil +} + +// filledByMoved says a filled path is a file of a module whose resources were moved into the node's own composition. +func filledByMoved(resources []map[string]json.RawMessage, path string, moved map[string]bool) bool { + for _, r := range resources { + var kind, at, id string + _ = json.Unmarshal(r["type"], &kind) + _ = json.Unmarshal(r["path"], &at) + _ = json.Unmarshal(r["id"], &id) + if kind == string(sdkdecl.TypeFile) && at == path { + if module, ok := sdkdecl.ModuleGuess(id); ok && moved[module] { + return true + } + } + } + return false +} + +// writeComposed writes one composition and its digest where they differ from what the buckets hold: the files by +// digest first, then the composition, then its digest and judgement, so a reader that finds the digest finds the +// composition and its files. It answers whether it wrote anything. +func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdelivery.Composed) (bool, error) { + c, objects, err := c.ByDigest() + if err != nil { + return false, err + } + if len(objects) > 0 { + if at.objects == nil { + return false, fmt.Errorf("%s holds a file over 64 KiB and %s's object bucket is not open", key, at.node) + } + named := make(map[string][]byte, len(objects)) + for digest, body := range objects { + name, err := sdkdecl.ObjectName(digest) + if err != nil { + return false, err + } + named[name] = body + } + if err := putObjects(ctx, at.objects, named); err != nil { + return false, err + } + } + digest, err := sdkdelivery.ComposedDigestOf(c) + if err != nil { + return false, err + } + wrote := false + held, revision, err := readComposed(ctx, at.bucket, key) + if err != nil { + return false, err + } + if heldDigest, _ := sdkdelivery.ComposedDigestOf(held); revision == 0 || heldDigest != digest { + signed, err := at.sign(c) + if err != nil { + return false, err + } + value, err := sdkdelivery.EncodeComposed(signed) + if err != nil { + return false, err + } + if err := casWrite(ctx, at.bucket, key, value, revision); err != nil { + return false, err + } + wrote = true + } + judged, err := judgeMove(ctx, at, key, c, digest) + if composeTrace != nil { + composeTrace(fmt.Sprintf("%s: %s: written %v, judged %s %q user-list-only %v against %.19s, err %v", at.node, key, wrote, + judged.Judgement, judged.Words, judged.UserListOnly, judged.Against, err)) + } + if err != nil { + return wrote, err + } + written, err := keepDigest(ctx, at.digests, key, judged) + return wrote || written, err +} + +// readComposed is the composition a bucket holds under key and its revision; revision 0 when there is none. +func readComposed(ctx context.Context, kv composedBucketsKV, key string) (sdkdelivery.Composed, uint64, error) { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return sdkdelivery.Composed{}, 0, nil + } + if err != nil { + return sdkdelivery.Composed{}, 0, err + } + c, err := sdkdelivery.DecodeComposed(e.Value()) + if err != nil { + // A value this controller cannot read is written over: it is derived, and composed again here. + return sdkdelivery.Composed{}, e.Revision(), nil + } + return c, e.Revision(), nil +} + +// keepDigest writes the digest and judgement beside a composition when it differs from what the bucket holds. +func keepDigest(ctx context.Context, kv composedBucketsKV, key string, v sdkdelivery.ComposedDigest) (bool, error) { + value, err := sdkdelivery.EncodeComposedDigest(v) + if err != nil { + return false, err + } + v.V = sdkdelivery.Version + var revision uint64 + e, err := kv.Get(ctx, key) + switch { + case errors.Is(err, jetstream.ErrKeyNotFound): + case err != nil: + return false, err + default: + revision = e.Revision() + if held, derr := sdkdelivery.DecodeComposedDigest(e.Value()); derr == nil && sdkdelivery.SameComposedDigest(held, v) { + return false, nil + } + } + return true, casWrite(ctx, kv, key, value, revision) +} + +// casWrite creates key when revision is 0 and otherwise replaces the revision read: a second writer that wrote in +// between makes this one fail, said, and the next event composes again. +func casWrite(ctx context.Context, kv composedBucketsKV, key string, value []byte, revision uint64) error { + var err error + if revision == 0 { + _, err = kv.Create(ctx, key, value) + } else { + _, err = kv.Update(ctx, key, value, revision) + } + if err != nil { + return fmt.Errorf("writing %s: %w", key, err) + } + return nil +} + +// writeNodeCompositions writes every composition of a node that differs, and prunes what is no longer kept. It answers +// the keys it wrote. +func writeNodeCompositions(ctx context.Context, at composedPlace, comps map[string]sdkdelivery.Composed) ([]string, error) { + keys := make([]string, 0, len(comps)) + for k := range comps { + keys = append(keys, k) + } + sort.Strings(keys) + var wrote []string + var errs []error + for _, key := range keys { + w, err := writeComposed(ctx, at, key, comps[key]) + if err != nil { + errs = append(errs, err) + continue + } + if w { + wrote = append(wrote, key) + } + } + if err := pruneCompositions(ctx, at); err != nil { + errs = append(errs, err) + } + return wrote, errors.Join(errs...) +} + +// pruneCompositions deletes, for each module of the node, the compositions outside its ComposedKeep newest fingerprints +// that neither its want nor its have names, each with its digest (ADR 0539 clause 4). Newest is by when each was last +// written: the controller writes only the fingerprint a module is registered at, so an older one is never written again. +func pruneCompositions(ctx context.Context, at composedPlace) error { + lister, err := at.bucket.ListKeys(ctx) + if err != nil { + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil + } + return err + } + type kept struct { + key, fp string + at time.Time + } + byModule := map[string][]kept{} + for key := range lister.Keys() { + node, module, fp, isNode, err := sdkdelivery.ParseComposedKey(key) + if err != nil || isNode || node != at.node { + continue + } + e, err := at.bucket.Get(ctx, key) + if err != nil { + continue + } + byModule[module] = append(byModule[module], kept{key, fp, e.Created()}) + } + var errs []error + for module, ks := range byModule { + if len(ks) <= sdkdelivery.ComposedKeep { + continue + } + sort.Slice(ks, func(i, j int) bool { return ks[i].at.After(ks[j].at) }) + wanted, err := at.wanted(ctx, module) + if err != nil { + errs = append(errs, err) + continue + } + have, _, err := at.ran(ctx, module) + if err != nil { + errs = append(errs, err) + continue + } + for _, k := range ks[sdkdelivery.ComposedKeep:] { + if k.fp == wanted || k.fp == have.FP { + continue + } + if err := at.bucket.Delete(ctx, k.key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + errs = append(errs, err) + continue + } + if err := at.digests.Delete(ctx, k.key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +// controllerSigner signs a composition with the controller's active key, which its identity store keeps and never hands +// out (as it signed declarations). +func controllerSigner(ctx context.Context, open *stores) (func(sdkdelivery.Composed) (sdkdelivery.Composed, error), error) { + ident, err := open.Identity(ctx) + if err != nil { + return nil, err + } + key, err := ident.Active(ctx) + if err != nil { + return nil, err + } + public := ed25519.PublicKey(key.Public) + return func(c sdkdelivery.Composed) (sdkdelivery.Composed, error) { + return sdkdelivery.SignComposedBy(c, public, func(m []byte) ([]byte, error) { return ident.Sign(ctx, m) }) + }, nil +} diff --git a/cmd/mesh-controller/composed_writer_test.go b/cmd/mesh-controller/composed_writer_test.go new file mode 100644 index 00000000..65efa8df --- /dev/null +++ b/cmd/mesh-controller/composed_writer_test.go @@ -0,0 +1,362 @@ +package main + +import ( + "compress/gzip" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "os" + "sort" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" +) + +// fakeComposedKV is a key-value bucket in memory: revisions, a creation time per write, history, compare-and-set. +type fakeComposedKV struct { + name string + rev uint64 + clock time.Time + history map[string][]fakeComposedEntry + writes int + deletes int +} + +type fakeComposedEntry struct { + bucket, key string + value []byte + rev uint64 + at time.Time + op jetstream.KeyValueOp +} + +func (e fakeComposedEntry) Bucket() string { return e.bucket } +func (e fakeComposedEntry) Key() string { return e.key } +func (e fakeComposedEntry) Value() []byte { return e.value } +func (e fakeComposedEntry) Revision() uint64 { return e.rev } +func (e fakeComposedEntry) Created() time.Time { return e.at } +func (e fakeComposedEntry) Delta() uint64 { return 0 } +func (e fakeComposedEntry) Operation() jetstream.KeyValueOp { return e.op } + +func newFakeComposedKV(name string) *fakeComposedKV { + return &fakeComposedKV{name: name, clock: time.Date(2026, 10, 24, 17, 0, 0, 0, time.UTC), history: map[string][]fakeComposedEntry{}} +} + +func (f *fakeComposedKV) last(key string) (fakeComposedEntry, bool) { + h := f.history[key] + if len(h) == 0 || h[len(h)-1].op != jetstream.KeyValuePut { + return fakeComposedEntry{}, false + } + return h[len(h)-1], true +} + +func (f *fakeComposedKV) put(key string, value []byte, op jetstream.KeyValueOp) uint64 { + f.rev++ + f.clock = f.clock.Add(time.Second) + f.history[key] = append(f.history[key], fakeComposedEntry{f.name, key, value, f.rev, f.clock, op}) + return f.rev +} + +func (f *fakeComposedKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + e, ok := f.last(key) + if !ok { + return nil, jetstream.ErrKeyNotFound + } + return e, nil +} + +func (f *fakeComposedKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { + if _, ok := f.last(key); ok { + return 0, jetstream.ErrKeyExists + } + f.writes++ + return f.put(key, value, jetstream.KeyValuePut), nil +} + +func (f *fakeComposedKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) { + e, ok := f.last(key) + if !ok || e.rev != revision { + return 0, fmt.Errorf("wrong last sequence") + } + f.writes++ + return f.put(key, value, jetstream.KeyValuePut), nil +} + +func (f *fakeComposedKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { + if _, ok := f.last(key); !ok { + return jetstream.ErrKeyNotFound + } + f.deletes++ + f.put(key, nil, jetstream.KeyValueDelete) + return nil +} + +func (f *fakeComposedKV) History(_ context.Context, key string, _ ...jetstream.WatchOpt) ([]jetstream.KeyValueEntry, error) { + h := f.history[key] + if len(h) == 0 { + return nil, jetstream.ErrKeyNotFound + } + out := make([]jetstream.KeyValueEntry, len(h)) + for i, e := range h { + out[i] = e + } + return out, nil +} + +type fakeLister struct{ keys chan string } + +func (l fakeLister) Keys() <-chan string { return l.keys } +func (l fakeLister) Stop() error { return nil } + +func (f *fakeComposedKV) ListKeys(_ context.Context, _ ...jetstream.WatchOpt) (jetstream.KeyLister, error) { + var keys []string + for k := range f.history { + if _, ok := f.last(k); ok { + keys = append(keys, k) + } + } + sort.Strings(keys) + ch := make(chan string, len(keys)) + for _, k := range keys { + ch <- k + } + close(ch) + return fakeLister{ch}, nil +} + +// fakeObjects is a node's object bucket in memory. +type fakeObjects map[string][]byte + +func (o fakeObjects) Has(_ context.Context, name string) (bool, bool, error) { + _, ok := o[name] + return ok, false, nil +} +func (o fakeObjects) Put(_ context.Context, name string, content []byte) error { + o[name] = content + return nil +} + +// aPlace is a node's buckets in memory, signed with a key of the test's, the want and the have given. +func aPlace(t *testing.T, node string) (composedPlace, ed25519.PublicKey, map[string]string, map[string]sdkdelivery.Have) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + wants, haves := map[string]string{}, map[string]sdkdelivery.Have{} + at := composedPlace{node: node, bucket: newFakeComposedKV("composed-" + node), digests: newFakeComposedKV("mesh-controller_composed"), + objects: fakeObjects{}, + wanted: func(_ context.Context, module string) (string, error) { return wants[module], nil }, + ran: func(_ context.Context, module string) (sdkdelivery.Have, bool, error) { + h, ok := haves[module] + return h, ok, nil + }, + policy: func(string) (string, string, string) { return catalogue.PolicyRoll, catalogue.FromDefault, "" }, + sign: func(c sdkdelivery.Composed) (sdkdelivery.Composed, error) { return sdkdelivery.SignComposed(c, priv) }, + } + return at, pub, wants, haves +} + +// capturedBody is a node's declaration as the controller's `plan` verb answered it on 2026-10-24 at 16:19, the body a +// send of declarationFor carries (testdata/declaration-.json.gz, copied from mesh-sdk's; the resources carrying +// a sealed value or a secret taken out), and a fingerprint standing in for each module's build. +func capturedBody(t *testing.T, node string) ([]byte, map[string]string) { + t.Helper() + f, err := os.Open("testdata/declaration-" + node + ".json.gz") + if err != nil { + t.Fatal(err) + } + defer f.Close() + z, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + raw, err := io.ReadAll(z) + if err != nil { + t.Fatal(err) + } + var d struct { + Builds map[string]string `json:"builds"` + } + if err := json.Unmarshal(raw, &d); err != nil { + t.Fatal(err) + } + fps := map[string]string{} + for m, b := range d.Builds { + fps[m] = sdkdelivery.SourcePrefix2 + fmt.Sprintf("%064x", []byte(b))[:64] + } + return raw, fps +} + +func resourceIDs(t *testing.T, raws []json.RawMessage) []string { + t.Helper() + var out []string + for _, r := range raws { + var head struct{ ID string } + if err := json.Unmarshal(r, &head); err != nil { + t.Fatal(err) + } + out = append(out, head.ID) + } + return out +} + +// 1945/AC1: for a captured node's declaration, the union of its module compositions and its own is the declaration's +// resources, each once; each module's at its fingerprint; a module with no fingerprint of the new form (the private +// network's mesh-wireguard, which nobody builds) rides in the node's own. +func TestTheCompositionsOfANodeAreItsDeclaration(t *testing.T) { + for _, node := range []string{"ace", "g14"} { + body, fps := capturedBody(t, node) + var d struct { + Resources []json.RawMessage `json:"resources"` + } + _ = json.Unmarshal(body, &d) + comps, moved, err := compositionsOf(node, body, fps, nil, nil) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + if strings.Join(moved, ",") != "mesh-wireguard" { + t.Errorf("%s: moved into the node's own %v, want mesh-wireguard", node, moved) + } + var union []string + for key, c := range comps { + _, module, fp, isNode, err := sdkdelivery.ParseComposedKey(key) + if err != nil || (!isNode && fp != fps[module]) { + t.Errorf("%s: %s is at %s (%v)", node, key, fp, err) + } + if _, err := sdkdelivery.EncodeComposed(c); err != nil { + t.Errorf("%s: %s: %v", node, key, err) + } + union = append(union, resourceIDs(t, c.Resources)...) + } + want := resourceIDs(t, d.Resources) + sort.Strings(union) + sort.Strings(want) + if strings.Join(union, " ") != strings.Join(want, " ") { + t.Errorf("%s: the compositions hold %d resources, the declaration %d, or they differ", node, len(union), len(want)) + } + own, _ := sdkdelivery.ComposedNodeKey(node) + if ids := strings.Join(resourceIDs(t, comps[own].Resources), " "); !strings.Contains(ids, "mesh-wireguard.") { + t.Errorf("%s: the node's own composition does not hold mesh-wireguard's resources: %s", node, ids) + } + } +} + +// 1945/AC1: a composition is written signed by the controller's key with its digest beside it; written again +// unchanged it writes nothing; a changed input writes a new revision of that key alone, by compare-and-set. +func TestACompositionIsWrittenOnceAndAgainOnlyWhenItChanges(t *testing.T) { + body, fps := capturedBody(t, "g14") + comps, _, err := compositionsOf("g14", body, fps, nil, nil) + if err != nil { + t.Fatal(err) + } + at, pub, _, _ := aPlace(t, "g14") + wrote, err := writeNodeCompositions(context.Background(), at, comps) + if err != nil || len(wrote) != len(comps) { + t.Fatalf("first write: %d of %d (%v)", len(wrote), len(comps), err) + } + for key := range comps { + e, err := at.bucket.Get(context.Background(), key) + if err != nil { + t.Fatal(err) + } + c, err := sdkdelivery.DecodeComposed(e.Value()) + if err != nil { + t.Fatal(err) + } + digest, err := sdkdelivery.VerifyComposed(c, []ed25519.PublicKey{pub}) + if err != nil { + t.Fatalf("%s: %v", key, err) + } + d, err := at.digests.Get(context.Background(), key) + if err != nil { + t.Fatal(err) + } + v, err := sdkdelivery.DecodeComposedDigest(d.Value()) + if err != nil || v.Digest != digest { + t.Errorf("%s: the digest beside it is %q, the composition's %s (%v)", key, v.Digest, digest, err) + } + } + writes := at.bucket.(*fakeComposedKV).writes + at.digests.(*fakeComposedKV).writes + if again, err := writeNodeCompositions(context.Background(), at, comps); err != nil || len(again) != 0 { + t.Errorf("written again unchanged: %v (%v)", again, err) + } + if now := at.bucket.(*fakeComposedKV).writes + at.digests.(*fakeComposedKV).writes; now != writes { + t.Errorf("an unchanged write wrote %d values", now-writes) + } + + key, _ := sdkdelivery.ComposedKey("g14", "fail2ban", fps["fail2ban"]) + changed := comps[key] + changed.Data = true + comps[key] = changed + wrote, err = writeNodeCompositions(context.Background(), at, comps) + if err != nil || len(wrote) != 1 || wrote[0] != key { + t.Errorf("a changed composition wrote %v (%v), want %s alone", wrote, err, key) + } +} + +// 1945/AC1: a module's compositions outside its five newest fingerprints are deleted with their digests, unless its +// want or its have names one. +func TestCompositionsOutsideTheFiveNewestAreDeleted(t *testing.T) { + at, _, wants, haves := aPlace(t, "ace") + file := json.RawMessage(`{"id":"m.f","type":"file","module":"m","path":"/etc/m","content":"x"}`) + fp := func(i int) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", i), 64) } + for i := 1; i <= 8; i++ { + c := sdkdelivery.Composed{Node: "ace", Module: "m", FP: fp(i), Resources: []json.RawMessage{file}} + key, _ := sdkdelivery.ComposedKey("ace", "m", fp(i)) + if _, err := writeNodeCompositions(context.Background(), at, map[string]sdkdelivery.Composed{key: c}); err != nil { + t.Fatal(err) + } + if i == 1 { + wants["m"] = fp(1) + } + if i == 2 { + haves["m"] = sdkdelivery.Have{FP: fp(2), Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + } + } + if err := pruneCompositions(context.Background(), at); err != nil { + t.Fatal(err) + } + for i := 1; i <= 8; i++ { + key, _ := sdkdelivery.ComposedKey("ace", "m", fp(i)) + _, err := at.bucket.Get(context.Background(), key) + _, derr := at.digests.Get(context.Background(), key) + kept := err == nil && derr == nil + wantKept := i >= 4 || i == 1 || i == 2 + if kept != wantKept { + t.Errorf("fingerprint %d kept %v, want %v (the wanted 1, the run 2, the five newest 4 to 8)", i, kept, wantKept) + } + } +} + +// 1945 (found by the served test): a module the node runs that composes no resource of its own still has its +// composition, empty, at its fingerprint, so a want can name it; a module the node runs with no fingerprint of the new +// form gets none. +func TestAModuleWithNoResourceOfItsOwnStillHasItsComposition(t *testing.T) { + body := []byte(`{"declaration":1,"resources":[]}`) + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64) + comps, _, err := compositionsOf("laptop", body, map[string]string{"board": fp}, nil, []string{"board", "legacy"}) + if err != nil { + t.Fatal(err) + } + key, _ := sdkdelivery.ComposedKey("laptop", "board", fp) + c, ok := comps[key] + if !ok || len(c.Resources) != 0 { + t.Fatalf("the board's composition: %+v (%v)", c, ok) + } + if _, err := sdkdelivery.EncodeComposed(c); err != nil { + t.Errorf("an empty composition does not encode: %v", err) + } + for k := range comps { + if strings.Contains(k, "legacy") { + t.Errorf("a module with no fingerprint of the new form was composed: %s", k) + } + } +} diff --git a/cmd/mesh-controller/conditions_brief_test.go b/cmd/mesh-controller/conditions_brief_test.go deleted file mode 100644 index 135c6336..00000000 --- a/cmd/mesh-controller/conditions_brief_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// stalledConditions are n open conditions like those of 2026-10-08: one per delivery held past its bound, -// each keeping all its evidence. -func stalledConditions(n int) []conditions.Condition { - at := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) - var out []conditions.Condition - for i := 0; i < n; i++ { - id := fmt.Sprintf("d-%012x", 0x51a11ed+i) - lines := stalledObservations([]stalledLine{{ID: id, State: "checking", For: "13h2m", Bound: "45m", - H2: "close: recheck", Says: "the check was asked and nothing has answered"}}) - o := lines[0] - c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Subject: conditions.Subject{Scope: o.Scope, ID: o.ID}, - Severity: o.Severity, Summary: o.Summary, Source: probeDeliveriesID, Raised: at, LastObserved: at, - Observations: 160, Count: 1, Resolver: conditions.ResolverSelf} - for e := 0; e < conditions.KeptEvidence; e++ { - c.Evidence = append(c.Evidence, conditions.Evidence{At: at.Add(time.Duration(e) * 5 * time.Minute), - Said: "checking for " + (13*time.Hour + time.Duration(e)*time.Minute).String()}) - } - out = append(out, c) - } - return out -} - -// novox/hq issue 314: a list of hundreds of open conditions carries each one's newest evidence, with -// how much more it keeps and where to read it, and the verb's answer says it once — as data, not again -// as the text the command printed. On 2026-10-08, 689 stalled deliveries made that answer more than the -// bus carries in one message; now it is less than half of one. -func TestAListOfConditionsCarriesEachInBrief(t *testing.T) { - list := stalledConditions(689) - printed, _ := json.MarshalIndent(map[string]any{"conditions": inBrief(list), "open": len(list), - "counted": counted(list)}, "", " ") - var parsed any - if err := json.Unmarshal(printed, &parsed); err != nil { - t.Fatal(err) - } - wire, _ := json.Marshal(map[string]any{"result": verbAnswer{OK: true, Answer: parsed, - Output: "its answer, as data, is `answer`\n"}}) - before, _ := json.MarshalIndent(map[string]any{"conditions": list}, "", " ") - t.Logf("689 stalled deliveries: %d bytes on the wire, where the whole list printed was %d and was sent twice", - len(wire), len(before)) - if len(wire) >= 512<<10 { - t.Fatalf("the answer of 689 conditions is %d bytes, not under half of one message of the bus", len(wire)) - } - var doc struct { - Conditions []conditionInBrief `json:"conditions"` - Counted map[string]int `json:"counted"` - } - if err := json.Unmarshal(printed, &doc); err != nil { - t.Fatal(err) - } - c := doc.Conditions[0] - if len(doc.Conditions) != 689 || len(c.Evidence) != 1 || c.EvidenceKept != conditions.KeptEvidence || - c.Key != list[0].Key || c.Summary != list[0].Summary || !strings.Contains(c.More, "key="+c.Key) { - t.Fatalf("a condition in brief reads %+v", c) - } - if doc.Counted["D14 stalled"] != 689 { - t.Fatalf("counted %v", doc.Counted) - } - if one := inBrief(list[:1])[0]; len(list[0].Evidence) != conditions.KeptEvidence || one.Evidence[0] != list[0].Evidence[0] { - t.Fatal("the brief changed the condition it was made from, or kept other than the newest") - } -} - -// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and -// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314). diff --git a/cmd/mesh-controller/container_groups_test.go b/cmd/mesh-controller/container_groups_test.go index 6fd9f9ba..7b154fc7 100644 --- a/cmd/mesh-controller/container_groups_test.go +++ b/cmd/mesh-controller/container_groups_test.go @@ -1,9 +1,6 @@ package main import ( - "io" - "os" - "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -18,22 +15,6 @@ const n8nWithAGroup = `{"module":"n8n","version":"1", "image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0", "groups":["${config:media-gid}"]}]}` -// printedBy runs reportUnhostable and returns what it printed. -func printedBy(t *testing.T, node string, plan catalogue.Resolution) string { - t.Helper() - stdout := os.Stdout - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - os.Stdout = w - reportUnhostable(node, plan) - w.Close() - os.Stdout = stdout - out, _ := io.ReadAll(r) - return string(out) -} - // 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named. func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) { m, err := catalogue.ParseManifest([]byte(n8nWithAGroup)) @@ -48,9 +29,6 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 { t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable) } - if out := printedBy(t, "ace", got); out != "" { - t.Errorf("a node that reads groups was told: %q", out) - } got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{}) if err != nil { @@ -59,7 +37,4 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing if len(got.Modules) != 0 || len(got.Unhostable) != 1 { t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable) } - if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") { - t.Errorf("the hold does not name the module and the capability: %q", out) - } } diff --git a/cmd/mesh-controller/core_box.go b/cmd/mesh-controller/core_box.go index 33c99192..a86d1f68 100644 --- a/cmd/mesh-controller/core_box.go +++ b/cmd/mesh-controller/core_box.go @@ -246,7 +246,7 @@ const kindVaultSilent = "vault-silent" // vaultSilentObservation is that, as a condition: `machine..vault-silent`. func vaultSilentObservation(node string, b coreBoxes) conditions.Observation { o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent, - Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource} + Machine: node, Severity: conditions.Warning, Source: "sends"} var said []string if len(b.notPlaced) > 0 { o.Severity = conditions.Urgent diff --git a/cmd/mesh-controller/core_downgrade.go b/cmd/mesh-controller/core_downgrade.go deleted file mode 100644 index cae82843..00000000 --- a/cmd/mesh-controller/core_downgrade.go +++ /dev/null @@ -1,215 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// **No send but a person's `push ` and a gate's own put-back moves the mesh's own core to a build older -// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a -// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under -// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans; -// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then -// sent the control node the controller build before it. The older binary read the row it could not serve and the -// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR -// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here, -// without needing its trigger: whatever made the mesh's record older, the send keeps the build. -// -// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last -// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or -// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend -// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by -// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until -// a send carries the record — a person's `push ` accepts the older build — or the record moves forward: a -// newer merge, or `rebuild mesh-controller`. -// -// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits' -// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's -// current build by the same order, and the guard does not stop that move. A build that failed its gate is never -// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own -// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back -// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's -// schema already (issue 352). - -// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record. -const kindCoreDowngrade = "core-downgrade-kept" - -// coreDowngradeSource is the condition's source: the sends, which raise and clear it. -const coreDowngradeSource = "sends" - -// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the -// store and its seat's row, and the node-engine, which reads what the controller declares. -var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine} - -// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate. -type askedBuild struct { - asked time.Time - failedGate bool -} - -// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when -// the request time is not known), and whether it failed its gate — and false when the records hold none. A -// variable, so a test reads its own order. -var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) { - builds, err := inv.Builds(ctx, module, askedLookBack) - if err != nil { - return askedBuild{}, false, err - } - for _, b := range builds { - if b.Worked() && sameCommit(b.Commit, commit) { - failed, err := inv.GateFailed(ctx, b.ID) - if err != nil { - return askedBuild{}, false, err - } - return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil - } - } - return askedBuild{}, false, nil -} - -// askedLookBack is how many of a module's newest builds are read for when a build was asked. -const askedLookBack = 50 - -// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record. -type coreDowngrade struct { - module, node, component string - // sent is the build the machine was last sent, record the one the mesh holds. - sent, record string - sentAsked, recordAsked time.Time -} - -func (d coreDowngrade) String() string { - return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+ - "keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+ - "(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record), - d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node, - d.module) -} - -// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine -// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move -// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure -// but for asked. -func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope, - asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) { - var out []coreDowngrade - for module, was := range sent { - component := coreComponent(module) - now := record[module] - if !guarded(component) || was == "" || now == "" || sameCommit(was, now) { - continue - } - if scope.puttingBack && scope.modules[module] { - continue - } - from, known, err := asked(module, was) - if err != nil { - return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err) - } - if !known || from.failedGate { - continue - } - to, known, err := asked(module, now) - if err != nil { - return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err) - } - if known && to.asked.Before(from.asked) { - out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now, - sentAsked: from.asked, recordAsked: to.asked}) - } - } - sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module }) - return out, nil -} - -func guarded(component string) bool { - for _, c := range downgradeGuarded { - if c == component { - return true - } - } - return false -} - -// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it -// was last sent, module → commit, each said and raised as a condition. -func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string, - current map[string]inventory.CurrentBuild) (map[string]string, error) { - record := map[string]string{} - for m, c := range current { - record[m] = c.Commit - } - found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) { - return buildAsked(ctx, inv, module, commit) - }) - if err != nil || len(found) == 0 { - return nil, err - } - out := map[string]string{} - for _, d := range found { - out[d.module] = d.sent - sayCoreDowngrade(ctx, d) - } - return out, nil -} - -// coreDowngradeObservation is a kept core build as a condition: `core...`. -func coreDowngradeObservation(component, node, summary string) conditions.Observation { - return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade, - Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource, - Summary: summary} -} - -func sayCoreDowngrade(ctx context.Context, d coreDowngrade) { - fmt.Printf("KEPT: %s\n", d) - keeper := conditionsFrom - if keeper == nil { - return - } - if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil { - fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err) - } -} - -// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the -// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record -// moved forward — or did not carry at all. A send that kept the newer build leaves it open. -func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) { - keeper := conditionsFrom - if keeper == nil || carried == nil { - return - } - var open []string - for _, c := range downgradeGuarded { - if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found { - open = append(open, c) - } - } - if len(open) == 0 { - return - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err) - return - } - for module, now := range current { - component := coreComponent(module) - if !guarded(component) { - continue - } - if was, has := carried[module]; has && !sameCommit(was, now.Commit) { - continue - } - key := coreDowngradeObservation(component, node, "").Key() - if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil { - fmt.Printf("the condition %s could not be cleared: %v\n", key, err) - } - } -} diff --git a/cmd/mesh-controller/data_integration_test.go b/cmd/mesh-controller/data_integration_test.go index 68247082..d121a20b 100644 --- a/cmd/mesh-controller/data_integration_test.go +++ b/cmd/mesh-controller/data_integration_test.go @@ -80,17 +80,18 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent( if _, err := assign(ctx, open, "laptop", "house"); err != nil { t.Fatal(err) } - // Sent, and applied: a holder is asked only once it has been (novox/hq issue 275). - for _, node := range []string{"laptop", "anchor"} { - pushedAndApplied(t, open, node) - } - conn := onATestBus(t) js, err := broker.Dial(testbus.URL(t)) if err != nil { t.Fatal(err) } t.Cleanup(js.Close) + // Applied, and running: a holder is asked only once its module's have says it runs (novox/hq issue 275, ADR 0539 §9). + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + for _, run := range [][2]string{{"laptop", "keeper"}, {"laptop", "house"}, {"anchor", "keeper"}} { + putRunningHave(t, js, run[0], run[1]) + } laptop, anchor := &fakeHolder{}, &fakeHolder{} laptop.serve(t, conn, "laptop") anchor.serve(t, conn, "anchor") diff --git a/cmd/mesh-controller/declaration_files_integration_test.go b/cmd/mesh-controller/declaration_files_integration_test.go index b554e115..82bdad8d 100644 --- a/cmd/mesh-controller/declaration_files_integration_test.go +++ b/cmd/mesh-controller/declaration_files_integration_test.go @@ -15,7 +15,6 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/testbus" ) @@ -34,14 +33,6 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) t.Fatal(err) } defer conn.Close() - nc, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}, - MaxMsgsPerSubject: 1}); err != nil { - t.Fatal(err) - } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() api, err := jetstream.New(conn) @@ -61,19 +52,8 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10))) } resources = append(resources, aFileResource("small", "tiny\n")) - inline := sendable{Resources: resources} byDigest := sendable{fileReferences: true, Resources: resources} - bus := link.OverNATS{Conn: conn, JS: nc} - body, err := inline.Body() - if err != nil { - t.Fatal(err) - } - if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil || - !strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") { - t.Fatalf("the inline declaration of more than 3 MiB: %v", err) - } - bucket, err := openBucket(ctx, conn, "novox") if err != nil { t.Fatal(err) @@ -92,18 +72,14 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) if err != nil { t.Fatal(err) } - if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil { - t.Fatalf("the declaration by reference: %v", err) + // What the body names is exactly the three objects. + namedSizes, err := namedObjects(refBody) + if err != nil || len(namedSizes) != 3 { + t.Fatalf("what the body names: %v %v", namedSizes, err) } - - // What the stream holds for the node is the reference form, and it names exactly the three objects. - latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"}) - if err != nil || len(latest) != 1 { - t.Fatalf("the latest declarations: %v %v", len(latest), err) - } - named, err := namedByLatest("novox", latest["novox"]) - if err != nil || len(named) != 3 { - t.Fatalf("what the latest declaration names: %v %v", named, err) + named := map[string]bool{} + for name := range namedSizes { + named[name] = true } novoxBucket, err := broker.DeclarationFilesBucketOf("novox") if err != nil { diff --git a/cmd/mesh-controller/declaration_files_test.go b/cmd/mesh-controller/declaration_files_test.go index c428d418..e96083ce 100644 --- a/cmd/mesh-controller/declaration_files_test.go +++ b/cmd/mesh-controller/declaration_files_test.go @@ -12,6 +12,7 @@ import ( "time" sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" @@ -55,7 +56,7 @@ func wireSizeOf(t *testing.T, s sendable) int { t.Fatal(err) } // The signature the controller's key makes is 64 bytes. - msg, err := link.Envelope(body, make([]byte, 64)) + msg, err := json.Marshal(link.Signed{Declaration: body, Signature: make([]byte, 64)}) if err != nil { t.Fatal(err) } @@ -243,54 +244,6 @@ func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696 } } -// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the -// controller's grant does, and takes the second. -type fallbackDelivery struct { - bodies [][]byte - first error -} - -func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil } -func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) { - f.bodies = append(f.bodies, body) - if len(f.bodies) == 1 && f.first != nil { - return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first) - } - return "digest", nil -} - -// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes -// with every content inline and is made; a send that fails for another reason is not retried inline. -func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5 - users := contentOf("users", 100<<10) - s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}} - f := &fallbackDelivery{first: errors.New("timeout")} - digests, err := deliver(t.Context(), f, "novox", []readyNode{s}) - if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 { - t.Fatalf("%v %v %d", digests, err, len(f.bodies)) - } - if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) { - t.Error("the first body should name the file by digest and the second carry it inline") - } - other := &fallbackDelivery{} - other.first = nil - failing := &failingDelivery{err: errors.New("the bus is down")} - if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 { - t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls) - } -} - -type failingDelivery struct { - err error - calls int -} - -func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil } -func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) { - f.calls++ - return "", f.err -} - // A bucket for the sweep. type sweepBucketFake struct { objects map[string]time.Time // zero: named at the last look @@ -354,22 +307,43 @@ func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // } } -// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and -// never "names none", which would mark an object a node needs. -func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) { - s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}} - body, _ := s.Body() - msg, _ := link.Envelope(body, []byte("sig")) - named, err := namedByLatest("novox", msg) - name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10)))) +// What a node's compositions name is read from the values of its bucket `composed-`: the files each carries by digest +// (ADR 0539 §4). An unreadable one is an error and never "names none", which would mark an object a node needs. +func TestTheObjectsACompositionNamesAreReadFromItsValue(t *testing.T) { + content := contentOf("a", 70<<10) + digest := sdkdecl.DigestOf([]byte(content)) + ref, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": digest, + "size": len(content)}) + small := json.RawMessage(`{"id":"m.g","type":"file","module":"m","path":"/etc/g","content":"x"}`) + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64) + value, err := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: fp, + Resources: []json.RawMessage{ref, small}}) + if err != nil { + t.Fatal(err) + } + named, err := namedByCompositions("novox", [][]byte{value}) + name, _ := sdkdecl.ObjectName(digest) if err != nil || len(named) != 1 || !named[name] { t.Fatalf("%v %v", named, err) } - if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 { - t.Fatalf("a declaration naming none: %v %v", none, err) + none, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "ace", Module: "m", FP: fp, Resources: []json.RawMessage{small}}) + if got, err := namedByCompositions("ace", [][]byte{none}); err != nil || len(got) != 0 { + t.Fatalf("a composition naming none: %v %v", got, err) } - if _, err := namedByLatest("novox", []byte("not json")); err == nil { - t.Fatal("an unreadable latest declaration was read as naming none") + if got, err := namedByCompositions("ace", nil); err != nil || len(got) != 0 { + t.Fatalf("a node with no composition: %v %v", got, err) + } + if _, err := namedByCompositions("novox", [][]byte{[]byte("not json")}); err == nil { + t.Fatal("an unreadable composition was read as naming none") + } + // A node's objects are those of every composition it keeps: the five newest fingerprints of a module name their own. + other := sdkdecl.DigestOf([]byte(contentOf("b", 80<<10))) + ref2, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": other, "size": 80 << 10}) + older, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64), + Resources: []json.RawMessage{ref2}}) + both, err := namedByCompositions("novox", [][]byte{value, older}) + if err != nil || len(both) != 2 { + t.Fatalf("two fingerprints of one module name %v (%v), want both objects", both, err) } } @@ -502,33 +476,3 @@ func whileHeld(node string) bool { return true } } - -// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The -// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the -// lock is held. -func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9 - const node = "lock-send-node" - bus := &aBusThatSays{published: make(chan string, 1)} - bus.heldAtPublish = func() bool { return whileHeld(node) } - send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}} - release := lockFilesOf(node) // a sweep is judging this node's bucket - done := make(chan error, 1) - go func() { - _, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`)) - done <- err - }() - select { - case n := <-bus.published: - t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n) - case <-time.After(150 * time.Millisecond): - } - release() - select { - case <-bus.published: - case <-time.After(5 * time.Second): - t.Fatal("the send did not go on once the sweep let go") - } - if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") { - t.Fatalf("the send did not reach the publication (a lock held at it): %v", err) - } -} diff --git a/cmd/mesh-controller/declaration_room.go b/cmd/mesh-controller/declaration_room.go index 2eb0b6df..d7b5b546 100644 --- a/cmd/mesh-controller/declaration_room.go +++ b/cmd/mesh-controller/declaration_room.go @@ -2,65 +2,10 @@ package main import ( "bytes" - "context" "encoding/json" - "fmt" "strings" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" ) -// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21 -// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone. -// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when -// the declaration is back under 75%. The limit is the connection's own, not a constant. - -// kindDeclarationNearLimit is a declaration at or near the bus's largest message. -const kindDeclarationNearLimit = "declaration-near-limit" - -// declarationRoomSource is the condition's source: the sends. -const declarationRoomSource = "sends" - -func declarationRoomObservation(node string, size, limit int64) conditions.Observation { - o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindDeclarationNearLimit, - Token: kindDeclarationNearLimit, Machine: node, Severity: conditions.Warning, Source: declarationRoomSource} - if size > limit { - o.Severity = conditions.Urgent - } - o.Summary = fmt.Sprintf("%s's declaration is %d bytes on the wire, %d%% of the bus's largest message of %d bytes", - node, size, size*100/limit, limit) - if size > limit { - o.Summary += ": over it, so it cannot be sent" - } - return o -} - -func init() { - link.DeclarationMeasured = func(ctx context.Context, node string, size, limit int64) { - settleDeclarationRoom(ctx, conditionsFrom, node, size, limit) - } -} - -// settleDeclarationRoom raises or clears the condition of a declaration's size. A nil keeper (a tool, not the -// serving controller) says nothing. -func settleDeclarationRoom(ctx context.Context, keeper *conditions.Keeper, node string, size, limit int64) { - if keeper == nil || limit <= 0 { - return - } - obs := declarationRoomObservation(node, size, limit) - if float64(size) < link.NearTheLimit*float64(limit) { - if _, err := keeper.Clear(ctx, obs.Key(), fmt.Sprintf("%s's declaration is %d bytes, under the limit's margin", node, size)); err != nil { - fmt.Printf("the condition %s could not be cleared: %v\n", obs.Key(), err) - } - return - } - fmt.Printf("NEAR THE LIMIT: %s\n", obs.Summary) - if _, err := keeper.Observe(ctx, obs); err != nil { - fmt.Printf("the condition %s could not be raised: %v\n", obs.Key(), err) - } -} - // compactKept writes the vault's export compact. The composition (the SDK's keptFile) writes it indented, and its 334 // sealed secrets were 164,763 bytes of it against 138,474 compact: a fifth of the declaration's room, in whitespace, // base64-encoded again on the wire. Every reader parses JSON, so the spacing is nobody's. A resource that is not diff --git a/cmd/mesh-controller/declaration_room_test.go b/cmd/mesh-controller/declaration_room_test.go deleted file mode 100644 index a52b547f..00000000 --- a/cmd/mesh-controller/declaration_room_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// The vault's export is sent compact (issue 1689): same document, no indentation, one rewrite. -func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) { - indented := "{\n \"export\": 1,\n \"kept\": [\n {\n \"name\": \"a\"\n }\n ]\n}\n" - other := "{\n \"x\": 1\n}\n" - in := []map[string]any{ - {"id": "mesh-vault.kept", "type": "file", "path": "/srv/vault/export.json", "mode": "0600", "content": indented}, - {"id": "mesh-vault.config", "type": "file", "path": "/srv/vault/config.json", "content": other}, - {"id": "other.kept", "type": "file", "path": "/srv/other/notes.json", "content": other}, - } - out := compactKept(in) - want := "{\"export\":1,\"kept\":[{\"name\":\"a\"}]}\n" - if out[0]["content"] != want || out[0]["mode"] != "0600" || out[0]["path"] != "/srv/vault/export.json" { - t.Fatalf("the export is %q (%v), expected %q with its mode and path kept", out[0]["content"], out[0], want) - } - if out[1]["content"] != other || out[2]["content"] != other { - t.Fatalf("a file that is not the export was rewritten: %v %v", out[1], out[2]) - } - if in[0]["content"] != indented { - t.Fatalf("the composed resource was changed in place") - } - if again := compactKept(out); again[0]["content"] != want { - t.Fatalf("compacting twice changed it: %q", again[0]["content"]) - } -} - -// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared. -func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) { - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }}) - t.Cleanup(func() { k.Close(t.Context()) }) - const limit = 1000 - key := declarationRoomObservation("novox", 0, limit).Key() - read := func() (conditions.Condition, bool) { - t.Helper() - c, found, err := conditions.ReadOne(t.Context(), store, key) - if err != nil { - t.Fatal(err) - } - return c, found - } - - settleDeclarationRoom(t.Context(), k, "novox", 749, limit) - if _, found := read(); found { - t.Fatal("a declaration at 74.9% raised a condition") - } - settleDeclarationRoom(t.Context(), k, "novox", 750, limit) - c, found := read() - if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") || - !strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" { - t.Fatalf("at 75%% the condition is %v %+v", found, c) - } - settleDeclarationRoom(t.Context(), k, "novox", 1001, limit) - if c, _ := read(); c.Severity != conditions.Urgent { - t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity) - } - settleDeclarationRoom(t.Context(), k, "novox", 700, limit) - if _, found := read(); found { - t.Fatal("the condition stayed after the declaration fell under 75%") - } -} diff --git a/cmd/mesh-controller/declaration_sweep.go b/cmd/mesh-controller/declaration_sweep.go index 33bc2a79..0f9430cf 100644 --- a/cmd/mesh-controller/declaration_sweep.go +++ b/cmd/mesh-controller/declaration_sweep.go @@ -9,23 +9,22 @@ import ( "strings" "time" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) -// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is -// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest -// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else -// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed); +// **An object is removed a day after no composition of the node names it** (novox/hq ADR 0486 §3, ADR 0539 §4, issue 1696), so a +// node that is asleep and applies its wants later still finds every object they name. What a node's compositions name is read +// from its bucket `composed-`, where the controller keeps each module's five newest fingerprints and the node's own: nothing +// else is kept of them. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed); // found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that -// node's latest declaration alone. Objects are immutable and regenerable from the -// controller's state, so a removal loses nothing. +// node's compositions alone. Objects are immutable and regenerable from the controller's state, so a removal loses nothing. -// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration. +// declarationUnnamed is the key of an object's metadata holding when it was first found named by no composition of the node. const declarationUnnamed = "unnamed-since" // sweepEvery is how often the bucket is judged: the removal is exact to within this. @@ -92,41 +91,124 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st return r, nil } -// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never -// "names none": an object the node still needs would be marked for removal. -func namedByLatest(node string, signed []byte) (map[string]bool, error) { - var s link.Signed - if err := json.Unmarshal(signed, &s); err != nil { - return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err) - } - names, err := namedObjects(s.Declaration) - if err != nil { - return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err) - } - named := make(map[string]bool, len(names)) - for name := range names { - named[name] = true +// namedByCompositions is the name of every object the compositions of a node name: the files over 64 KiB each carries by +// digest (ADR 0539 §4). A composition that cannot be read is an error, never "names none": an object the node still needs +// would be marked for removal. +func namedByCompositions(node string, values [][]byte) (map[string]bool, error) { + named := map[string]bool{} + for _, value := range values { + c, err := sdkdelivery.DecodeComposed(value) + if err != nil { + return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err) + } + body, err := json.Marshal(map[string]any{"resources": c.Resources}) + if err != nil { + return nil, err + } + names, err := namedObjects(body) + if err != nil { + return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err) + } + for name := range names { + named[name] = true + } } return named, nil } -// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has -// no entry. -func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) { - js, err := conn.JetStream(nats.Context(ctx)) +// keptCompositionsOf are the values of every composition in a node's bucket `composed-` a node may still apply: +// each key's newest revision (each module's five newest fingerprints and the node's own, ADR 0539 §4), and any older +// revision of it whose digest a want of the node names, since the node applies exactly the composition its want names +// (clause 1) and a want may still name a revision a newer one replaced (the review's C1). A node with no bucket has none. +func keptCompositionsOf(ctx context.Context, js jetstream.JetStream, node string) ([][]byte, error) { + bucket, err := sdkdelivery.ComposedBucketFor(node) if err != nil { return nil, err } - out := map[string][]byte{} - for _, node := range nodes { - msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx)) - if errors.Is(err, nats.ErrMsgNotFound) { + kv, err := js.KeyValue(ctx, bucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("the compositions of %s: %w", node, err) + } + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("the compositions of %s: %w", node, err) + } + wanted, err := digestsWantedOn(ctx, js, node) + if err != nil { + return nil, err + } + var out [][]byte + for _, key := range keys { + history, err := kv.History(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } if err != nil { - return nil, fmt.Errorf("the latest declaration of %s: %w", node, err) + return nil, fmt.Errorf("the composition %s: %w", key, err) + } + out = append(out, keptRevisions(history, wanted)...) + } + return out, nil +} + +// keptRevisions are, of one key's history oldest first, the newest value when the key is not deleted, and every older +// value whose composition's digest is wanted. +func keptRevisions(history []jetstream.KeyValueEntry, wanted map[string]bool) [][]byte { + var out [][]byte + for i, e := range history { + if e.Operation() != jetstream.KeyValuePut { + continue + } + if i == len(history)-1 { + out = append(out, e.Value()) + continue + } + c, err := sdkdelivery.DecodeComposed(e.Value()) + if err != nil { + continue + } + if d, err := sdkdelivery.ComposedDigestOf(c); err == nil && wanted[d] { + out = append(out, e.Value()) + } + } + return out +} + +// digestsWantedOn is the digest of every composition a want of the node names, its own want's among them; none when +// the assigner's bucket is not there yet. +func digestsWantedOn(ctx context.Context, js jetstream.JetStream, node string) (map[string]bool, error) { + kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return map[string]bool{}, nil + } + if err != nil { + return nil, fmt.Errorf("the wants of %s: %w", node, err) + } + lister, err := kv.ListKeys(ctx, jetstream.MetaOnly()) + if err != nil { + return nil, fmt.Errorf("the wants of %s: %w", node, err) + } + out := map[string]bool{} + for key := range lister.Keys() { + e, err := kv.Get(ctx, key) + if err != nil { + continue + } + if n, _, err := sdkdelivery.ParseWantKey(key); err == nil && n == node { + if w, err := sdkdelivery.DecodeWant(e.Value()); err == nil && w.Composed != "" { + out[w.Composed] = true + } + } else if n, err := sdkdelivery.ParseWantNodeKey(key); err == nil && n == node { + if w, err := sdkdelivery.DecodeNodeWant(e.Value()); err == nil { + out[w.Composed] = true + } } - out[node] = msg.Data } return out, nil } @@ -259,15 +341,13 @@ func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, e return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err) } defer lockFilesOf(node)() - latest, err := latestDeclarations(working, conn, []string{node}) + values, err := keptCompositionsOf(working, js, node) if err != nil { return filesSweep{}, err } - named := map[string]bool{} - if body, ok := latest[node]; ok { - if named, err = namedByLatest(node, body); err != nil { - return filesSweep{}, err - } + named, err := namedByCompositions(node, values) + if err != nil { + return filesSweep{}, err } return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now()) } diff --git a/cmd/mesh-controller/declaration_sweep_wanted_test.go b/cmd/mesh-controller/declaration_sweep_wanted_test.go new file mode 100644 index 00000000..c2434fc2 --- /dev/null +++ b/cmd/mesh-controller/declaration_sweep_wanted_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" +) + +// 1945 (the review's C1): the sweep keeps the files of a key's newest composition and of every older revision a want +// of the node still names, so a node told to apply a replaced revision finds its files; an older revision no want names, +// and a deleted key, keep nothing. +func TestTheSweepKeepsWhatAWantStillNames(t *testing.T) { + revision := func(content string, rev uint64, op jetstream.KeyValueOp) fakeComposedEntry { + c := sdkdelivery.Composed{Node: "ace", Module: "m", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64), + Resources: []json.RawMessage{json.RawMessage(`{"id":"m.f","type":"file","module":"m","path":"/etc/m","content":"` + content + `"}`)}} + raw, err := sdkdelivery.EncodeComposed(c) + if err != nil { + t.Fatal(err) + } + return fakeComposedEntry{key: "k", value: raw, rev: rev, op: op} + } + digestOfEntry := func(e fakeComposedEntry) string { + c, _ := sdkdelivery.DecodeComposed(e.value) + d, _ := sdkdelivery.ComposedDigestOf(c) + return d + } + one, two, three := revision("one", 1, jetstream.KeyValuePut), revision("two", 2, jetstream.KeyValuePut), revision("three", 3, jetstream.KeyValuePut) + history := []jetstream.KeyValueEntry{one, two, three} + kept := keptRevisions(history, map[string]bool{digestOfEntry(one): true}) + if len(kept) != 2 || string(kept[0]) != string(one.value) || string(kept[1]) != string(three.value) { + t.Errorf("kept %d revisions, want the wanted first and the newest", len(kept)) + } + if kept := keptRevisions(history, map[string]bool{}); len(kept) != 1 { + t.Errorf("with no want, kept %d revisions, want the newest alone", len(kept)) + } + deleted := []jetstream.KeyValueEntry{one, fakeComposedEntry{key: "k", rev: 2, op: jetstream.KeyValueDelete}} + if kept := keptRevisions(deleted, map[string]bool{}); len(kept) != 0 { + t.Errorf("a deleted key kept %d revisions", len(kept)) + } +} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go deleted file mode 100644 index d626ad10..00000000 --- a/cmd/mesh-controller/delivery.go +++ /dev/null @@ -1,1213 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "slices" - "sort" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The controller's part in a delivery (novox/hq ADR 0239, to-be 47). -// -// **A delivery is mesh-delivery's; the walk is the controller's.** A delivery is one commit in one -// repository from its pull request's head to every machine, and a delivery group a few of them sharing a -// branch name; their states, order, holds and record belong to the module holding the `mesh-delivery` -// seat. The controller keeps what it already owned — the planner, the gate, registration, sending, the -// rollback — and the **walk**: one trunk commit's plan, tier by tier across the machines, one machine -// first and judged at the gate (ADR 0236). What changes here is when a walk starts. -// -// - A walk that moves the controller, the node-engine, the node tools, the bus or mesh-delivery itself is -// **on the controller's own path**: started by the merge, as every plan was. mesh-delivery cannot gate -// or deliver itself, and nothing the core needs to be repaired may wait for it. -// - Any other walk, **while the seat has a holder on record**, is opened by the merge and waits — nothing -// asked, nothing registered, nothing sent — until mesh-delivery says `deliver`, or a person says -// `plans go`. Nothing of it is registered before then, so no other send can carry it to a machine -// before its turn (ADR 0236 §4a carries everything registered). -// - With no holder on record, every walk starts at the merge, exactly as before this existed. -// -// The verbs mesh-delivery asks with are here: `delivery plan` (the planner's one answer for a diffset), -// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery -// go`, `delivery stop` and `delivery walks`. - -// onTheControllersPath are the modules, by name, whose walk never waits for the delivery's owner, and what each is. -// The delivery's owner is not among them: it is whichever module claims the seat (ownPathOf, novox/hq ADR 0437 §3). -var onTheControllersPath = map[string]string{ - "mesh-controller": "the controller", - "mesh-host": "the node-engine", - "node-tools": "the node tools", - "nats": "the bus", -} - -// claimantsOf are the modules among the catalogued entries that claim a seat, each once, in name order. -func claimantsOf(entries []inventory.Entry, seat string) []string { - var out []string - for _, e := range entries { - if e.Manifest.ClaimsSeat(seat) && !slices.Contains(out, e.Manifest.Module) { - out = append(out, e.Manifest.Module) - } - } - sort.Strings(out) - return out -} - -// ownPathOf is the controller's own path among these entries: the modules by name, and the modules that claim -// the delivery seat, which is how the delivery's owner is found (novox/hq ADR 0437 §3). -func ownPathOf(entries []inventory.Entry) map[string]string { - own := make(map[string]string, len(onTheControllersPath)+1) - for m, what := range onTheControllersPath { - own[m] = what - } - for _, m := range claimantsOf(entries, catalogue.DeliverySeat) { - own[m] = "the delivery's owner" - } - return own -} - -// onTheOwnPath is whether one module, with its manifest at hand, is on the controller's own path. -func onTheOwnPath(module string, m catalogue.Manifest) bool { - _, named := onTheControllersPath[module] - return named || m.ClaimsSeat(catalogue.DeliverySeat) -} - -// ownPathNow is ownPathOf the catalogue as it is. -func ownPathNow(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) { - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil, err - } - return ownPathOf(entries), nil -} - -// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a -// holder on record. Read from the catalogue the merge handler already holds; never from whether the -// holder answers, so a walk does not start by itself because mesh-delivery is down — that wait is said. -func deliverySeatHeld(entries []inventory.Entry) bool { - for _, e := range entries { - if len(e.On) > 0 && e.Manifest.ClaimsSeat(catalogue.DeliverySeat) { - return true - } - } - return false -} - -// awaitsFor is what a new walk waits for: nil when it starts at once — no holder on record, or a module -// on the controller's own path among those it moves. -func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDelivery { - if !deliverySeatHeld(entries) { - return nil - } - onPath := ownPathOf(entries) - for _, m := range modules { - if _, own := onPath[m]; own { - return nil - } - } - return &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat} -} - -// waitingNote is what a walk waiting for its word says, wherever it is read. -func waitingNote(p inventory.Plan) string { - return fmt.Sprintf("published; its walk waits for %s's word — `plans go %s --why …` starts it by hand", - p.Delivery.Awaits, p.ID) -} - -// waitingSaid is what a walk waiting for its word says when the delivery's owner said what holds it (novox/hq -// issue 529): a delivery held for a person, its reason and the verb that releases it — the person's decision, not -// a hand start — or the deliveries not at their turn. Told nothing, it is waitingNote. -func waitingSaid(p inventory.Plan, lines []stalledLine) string { - if len(lines) == 0 { - return waitingNote(p) - } - var held, unheld, other []string - for _, l := range lines { - switch l.State { - case "held": - held = append(held, l.Says) - case "unheld": - unheld = append(unheld, l.Says) - default: - other = append(other, l.Says) - } - } - switch { - case len(held) > 0: - return fmt.Sprintf("published; its walk waits for %s's word, and a delivery it carries is held for a person: %s", - p.Delivery.Awaits, strings.Join(held, "; ")) - case len(other) > 0: - // Not at its turn: it goes when its turn comes. A hand start here would skip its group's order (review of - // novox/hq issue 529), so none is offered. - return fmt.Sprintf("published; its walk waits for %s's word: %s", p.Delivery.Awaits, strings.Join(other, "; ")) - } - // Held by nothing: nothing will give it its word, so the hand start is the way on. - return fmt.Sprintf("published; its walk waits for %s's word: %s — `plans go %s --why …` starts it by hand", - p.Delivery.Awaits, strings.Join(unheld, "; "), p.ID) -} - -// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its -// first tier. Refused for a walk that does not wait. -func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) { - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return inventory.Plan{}, err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return inventory.Plan{}, err - } - switch { - case !p.Open(): - return p, fmt.Errorf("%s is %s: there is no walk to start", p.ID, p.State) - case p.Delivery == nil || p.Delivery.Awaits == "": - return p, fmt.Errorf("%s waits for nobody: it started at its merge", p.ID) - case p.Delivery.Go != nil: - return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By, - p.Delivery.Go.Local().Format("15:04:05")) - } - // **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): the word is - // taken, and the walk starts once no started walk it is related to is open (advanceOnce), so the delivery's - // owner says it once and is not refused. - now := time.Now().UTC() - p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why - p.Note = "let go by " + by + "; its first tier is asked next" - if err := inv.SavePlan(ctx, &p); err != nil { - return p, err - } - return p, nil -} - -// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still -// builds and registers; nothing further is asked or sent. -func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) { - p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC()) - return p, err -} - -// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the -// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk -// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere -// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules. -type stopAnswer struct { - Stopped string `json:"stopped"` - WalkedAgainBy string `json:"walked_again_by,omitempty"` - WalkedAgain []string `json:"walked_again,omitempty"` - LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"` - Kept []inventory.PlanKept `json:"kept,omitempty"` - Withheld []string `json:"withheld,omitempty"` - StillCarriedBy []string `json:"still_carried_by,omitempty"` -} - -// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it -// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending -// it for one group member ended every other delivery on it. -// -// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each -// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge. -// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older -// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk -// withholds the left-out merges' modules where it would build them only as dependents. -// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch -// goes on without them; with nothing left in it, the batch itself is stopped. -// - The stopped record, the batch and every merge moved are written in one transaction (decision 7). -// -// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not -// answer refuses the stop whole. -func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string, - now time.Time) (inventory.Plan, stopAnswer, error) { - said := stopAnswer{Stopped: id} - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return inventory.Plan{}, said, err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return inventory.Plan{}, said, err - } - batch := p.Batch() - if !p.Open() && !(batch && len(without) > 0) { - return p, said, fmt.Errorf("%s is already %s", p.ID, p.State) - } - var merges, again []inventory.BatchedMerge - if len(without) > 0 { - if merges, err = inv.MergesOf(ctx, p.ID); err != nil { - return p, said, err - } - if again, said.LeftOut, err = leaveOut(merges, without); err != nil { - return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) - } - } - stopped := func(r *inventory.Plan) { - if r.Delivery == nil { - r.Delivery = &inventory.PlanDelivery{} - } - r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut - r.State = inventory.PlanFailed - r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why) - } - withheld := movesOf(leftOutMerges(merges, said.LeftOut)) - said.Withheld = withheld - if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil { - return p, said, err - } - if batch { - return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return p, said, err - } - running := map[string][]string{} - for _, e := range entries { - running[e.Manifest.Module] = e.On - } - said.Kept = keptOf(p, movesOf(again), running) - stopped(&p) - p.Delivery.Kept = said.Kept - if len(again) == 0 { - if len(said.LeftOut) > 0 { - p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut)) - } - return p, said, inv.SavePlan(ctx, &p) - } - target, folded, foldMoves, err := batchToWalkAgain(ctx, inv, p.ID, again, now) - if err != nil { - return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) - } - target.Delivery.Batch.Closed = true - target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld) - sort.Strings(target.Delivery.Batch.Withheld) - said.WalkedAgainBy = target.ID - var moves []inventory.MergeMove - for _, m := range again { - said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) - moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone}) - } - p.Delivery.WalkedAgainBy = target.ID - p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", ")) - if len(said.LeftOut) > 0 { - p.Note += "; left out " + mergesLeftOutWords(said.LeftOut) - } - if len(said.Kept) > 0 { - p.Note += "; " + keptWords(said.Kept) - } - // The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7) — the - // batches folded into it too. - if err := inv.SavePlansMoving(ctx, append([]*inventory.Plan{&target, &p}, folded...), - append(moves, foldMoves...)); err != nil { - return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err) - } - // What the batch holds and names now: written again by the next look should this fail. - if err := keepBatch(ctx, inv, &target, now, ""); err != nil { - fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err) - } - fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID, - strings.Join(said.WalkedAgain, ", ")) - return p, said, nil -} - -// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which -// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped -// itself. -func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge, - said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error { - if len(again) == 0 { - stopped(b) - b.Delivery.Batch = nil - b.Note = "a batch whose every merge was left out by a stop: " + b.Note - return inv.SavePlan(ctx, b) - } - left := leftOutMerges(merges, said.LeftOut) - first := left[0] - r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, - Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{}, - Modules: map[string]*inventory.PlanModule{}} - if r.ID == b.ID { - r.ID += "-left" - } - stopped(&r) - r.Delivery.WalkedAgainBy = b.ID - r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID, - mergesLeftOutWords(said.LeftOut), b.ID) - b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld) - sort.Strings(b.Delivery.Batch.Withheld) - var moves []inventory.MergeMove - for _, m := range left { - moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone}) - } - for _, m := range again { - said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) - } - said.Stopped, said.WalkedAgainBy = r.ID, b.ID - if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil { - return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err) - } - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err) - } - fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID) - return nil -} - -// batchToWalkAgain is the batch a stopped walk's merges join (novox/hq ADR 0299): a batch of their kind not yet cut -// that they are related to (ADR 0304 decision 1, ADR 0306 decision 3), or a new one, not yet written. Related to -// two or more, the newer are folded into the oldest: the plans and merge moves that fold them, to be written in -// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds. -func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge, - now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) { - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - return inventory.Plan{}, nil, nil, err - } - own := false - onPath := ownPathOf(entries) - for _, n := range movesOf(again) { - if _, on := onPath[n]; on { - own = true - } - } - joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges) - if err != nil { - return inventory.Plan{}, nil, nil, err - } - if len(joins) > 0 { - into := joins[0] - if len(joins) == 1 { - return into, nil, nil, nil - } - plans, moves, err := foldInto(ctx, inv, &into, joins[1:]) - if err != nil { - return inventory.Plan{}, nil, nil, err - } - return into, plans[1:], moves, nil - } - first := again[0] - return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, - Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling, - Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil, nil, nil -} - -// leaveOut splits a walk's merges by the merges named to leave out, each `@` (a commit's -// first characters will do): the merges walked again, and those left out — each named one; each later merge of -// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is -// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is -// refused. -func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) { - var stopped []inventory.BatchedMerge - for _, w := range without { - repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@") - if !ok || repository == "" || len(commit) < 7 { - return nil, nil, fmt.Errorf("%q names no merge: @", w) - } - i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool { - return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit) - }) - if i < 0 { - return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit) - } - stopped = append(stopped, merges[i]) - } - same := func(a, b inventory.BatchedMerge) bool { - return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit - } - out := map[int]inventory.PlanLeftOut{} - for i, m := range merges { - if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) { - out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit} - } - } - for grew := true; grew; { - grew = false - for i, m := range merges { - if _, left := out[i]; left { - continue - } - for j, o := range out { - l := merges[j] - contains := o.Contains - if contains == "" { - contains = l.Commit - } - if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) { - out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains} - } else if shared := sharedMoves(m, l); len(shared) > 0 { - out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains, - Shares: shared} - } else { - continue - } - grew = true - break - } - } - } - var again []inventory.BatchedMerge - var left []inventory.PlanLeftOut - for i, m := range merges { - if o, is := out[i]; is { - left = append(left, o) - } else { - again = append(again, m) - } - } - sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" }) - return again, left, nil -} - -// sharedMoves are the modules two kept merges both moved when they were heard. -func sharedMoves(a, b inventory.BatchedMerge) []string { - var out []string - theirs := announcedOf(b).Moves - for _, n := range announcedOf(a).Moves { - if slices.Contains(theirs, n) && !slices.Contains(out, n) { - out = append(out, n) - } - } - sort.Strings(out) - return out -} - -// leftOutMerges are the kept merges a stop left out. -func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge { - var left []inventory.BatchedMerge - for _, m := range merges { - if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { - return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit - }) { - left = append(left, m) - } - } - return left -} - -// movesOf is every module kept merges moved when they were heard, sorted. -func movesOf(merges []inventory.BatchedMerge) []string { - var out []string - for _, m := range merges { - out = union(out, announcedOf(m).Moves) - } - sort.Strings(out) - return out -} - -// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached: -// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it -// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is -// built by it) reached every machine running it. -func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept { - var out []inventory.PlanKept - for name, m := range p.Modules { - if m == nil || slices.Contains(walkedAgain, name) { - continue - } - var machines []string - if m.FirstAt != nil || m.SentAt != nil { - machines = union(machines, m.First) - } - if m.SentAt != nil && len(m.First) == 0 { - machines = union(machines, running[name]) - } - if m.Gate != nil && m.Gate.Since != nil { - machines = union(machines, m.Gate.Machines) - } - for machine := range m.Rest { - machines = union(machines, []string{machine}) - } - if len(machines) == 0 { - continue - } - sort.Strings(machines) - out = append(out, inventory.PlanKept{Module: name, Machines: machines}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - -// keptWords is what a stopped walk leaves on the machines, as a person reads it. -func keptWords(kept []inventory.PlanKept) string { - var words []string - for _, k := range kept { - words = append(words, k.Module+" on "+strings.Join(k.Machines, ", ")) - } - return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ") -} - -// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped -// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a -// module it moves. Holding a merged change back for good is a revert's (ADR 0299). -func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge, - out []inventory.PlanLeftOut, withheld []string) ([]string, error) { - var named []inventory.BatchedMerge - for _, m := range leftOutMerges(merges, out) { - if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) { - named = append(named, m) - } - } - if len(named) == 0 { - return nil, nil - } - open, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - batches, err := inv.Batches(ctx) - if err != nil { - return nil, err - } - var carried []string - for _, p := range append(open, batches...) { - if p.ID == id { - continue - } - ms, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return nil, err - } - for _, m := range ms { - later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool { - return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s) - }) - shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) }) - if later || shares { - carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID)) - } - } - } - sort.Strings(carried) - return carried, nil -} - -// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or -// shares its modules, says so. -func mergesLeftOutWords(out []inventory.PlanLeftOut) string { - var words []string - for _, o := range out { - s := o.Repository + "@" + short(o.Commit) - switch { - case len(o.Shares) > 0: - s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)" - case o.Contains != "": - s += " (which contains " + short(o.Contains) + ")" - } - words = append(words, s) - } - return strings.Join(words, ", ") -} - -// orderMember is one member of a group, as mesh-delivery says it. -type orderMember struct { - ID string `json:"id"` - Repository string `json:"repository"` - Base string `json:"base,omitempty"` - Head string `json:"head,omitempty"` - Number int `json:"number,omitempty"` - Paths []string `json:"paths,omitempty"` - PathsTruncated bool `json:"paths_truncated,omitempty"` - Removed []string `json:"removed,omitempty"` - ModuleDirs []string `json:"module_dirs,omitempty"` - ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` - CloneURL string `json:"clone_url,omitempty"` - // BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit - // a verdict no build judges names when the head is checked again through mesh-delivery. - BaseSHA string `json:"base_sha,omitempty"` - // Ready says mesh-delivery wants the member's check queued before every other check (novox/hq ADR 0406 §3, issue - // 974). There is no ready mark any more (issue 1022): mesh-delivery says it when it asks for the Approve. - Ready bool `json:"ready,omitempty"` -} - -// pull is the member as the forge announced it. -func (m orderMember) pull() link.PullUpdated { - owner, repo, _ := strings.Cut(m.Repository, "/") - base := m.Base - if base == "" { - base = "main" - } - return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA, - CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed, - ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid, Ready: m.Ready} -} - -// orderPair is one "before" among a group's members, why, and the rules it won over. -type orderPair struct { - Before string `json:"before"` - After string `json:"after"` - Why string `json:"why"` - // Over is the rules that ordered the pair the other way and lost to Why, by precedence (novox/hq ADR - // 0249): said, so a person reading the order sees which rule won. - Over []string `json:"over,omitempty"` -} - -// orderReach is what a member moves, as the planner says. -type orderReach struct { - Moved []string `json:"moved,omitempty"` - Dependents []string `json:"dependents,omitempty"` - New []string `json:"new,omitempty"` - Manifests []string `json:"manifests,omitempty"` -} - -// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one. The -// order is read from the module graph and the manifests' change alone (novox/hq ADR 0419): a cycle is the only -// refusal, and CycleSaid names every module and edge of it. Edges are the module edges the pairs come from. -type groupOrder struct { - Order []string `json:"order"` - Pairs []orderPair `json:"pairs,omitempty"` - Cycle []string `json:"cycle,omitempty"` - CycleSaid string `json:"cycle_said,omitempty"` - Edges []inventory.Edge `json:"edges,omitempty"` - Reach map[string]orderReach `json:"reach,omitempty"` -} - -// orderBuiltBy is the reason a member goes before another that moves a module built by, standing on, packaging or -// declaring one of its modules: the module graph's own edge. -const orderBuiltBy = "built by" - -// orderOf is a group's order (novox/hq ADR 0419), pure: a member goes before another when a module it moves goes -// before a module the other moves by an edge of the module graph (built by), or by a rule of delivery_edges.go -// (verb before caller, version skew, engine before controller), which changes names the manifests' added verbs. -// No `after:` line is read. Ties break by repository then id, so every reading gives one order. A cycle through -// two members or more is named by its members, and none of them is ordered. -func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge, changes map[string]moduleChange) groupOrder { - out := groupOrder{Reach: reach} - movedBy := map[string][]string{} - var all []string - for _, m := range members { - for _, n := range reach[m.ID].Moved { - movedBy[n] = append(movedBy[n], m.ID) - all = appendNew(all, n) - } - } - add := func(before, after, why string) { - if before == after { - return - } - for _, p := range out.Pairs { - if p.Before == before && p.After == after && p.Why == why { - return - } - } - out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why}) - } - for _, e := range edges { - if !orders(e) { - continue - } - for _, before := range movedBy[e.To] { - for _, after := range movedBy[e.From] { - add(before, after, orderBuiltBy) - } - } - } - order := orderEdges(all, changes, edges, false) - out.Edges = order.Edges - out.CycleSaid = order.CycleSaid() - for _, e := range order.Edges { - for _, before := range movedBy[e.To] { - for _, after := range movedBy[e.From] { - add(before, after, e.Why) - } - } - } - sort.Slice(out.Pairs, func(i, j int) bool { - if out.Pairs[i].Before != out.Pairs[j].Before { - return out.Pairs[i].Before < out.Pairs[j].Before - } - if out.Pairs[i].After != out.Pairs[j].After { - return out.Pairs[i].After < out.Pairs[j].After - } - return out.Pairs[i].Why < out.Pairs[j].Why - }) - // Kahn's walk, the next always the first by repository and id among those with nothing before them. - waiting := map[string]int{} - for _, m := range members { - waiting[m.ID] = 0 - } - for _, p := range out.Pairs { - waiting[p.After]++ - } - done := map[string]bool{} - for len(done) < len(members) { - var ready []orderMember - for _, m := range members { - if !done[m.ID] && waiting[m.ID] == 0 { - ready = append(ready, m) - } - } - if len(ready) == 0 { - for _, m := range members { - if !done[m.ID] { - out.Cycle = append(out.Cycle, m.ID) - } - } - sort.Strings(out.Cycle) - return out - } - sort.Slice(ready, func(i, j int) bool { - if ready[i].Repository != ready[j].Repository { - return ready[i].Repository < ready[j].Repository - } - return ready[i].ID < ready[j].ID - }) - next := ready[0] - done[next.ID] = true - out.Order = append(out.Order, next.ID) - for _, p := range out.Pairs { - if p.Before == next.ID { - waiting[p.After]-- - } - } - } - return out -} - -// reachOfMembers is each member's reach, as the planner says it. -func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge) map[string]orderReach { - out := map[string]orderReach{} - for _, m := range members { - s := pullScope(m.pull(), entries, read, edges) - out[m.ID] = orderReach{Moved: s.Modules, Dependents: s.Dependents, New: s.New, Manifests: s.Manifests} - } - return out -} - -// changeMember is one change as the planner is asked about it, the same for `delivery plan` and `plans --what-if`. -func changeMember(repository, base, head string, paths, removed []string, moduleDirs *[]string) orderMember { - m := orderMember{ID: repository + "@" + head, Repository: repository, Base: base, Head: head, - Paths: paths, Removed: removed} - if moduleDirs != nil { - m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true - } - return m -} - -// deliveryPlanDocument is the answer of `delivery plan` and of `plans --what-if … --data`: one change's -// change plan with its reach, the mesh's own part in it and whether the gate would run. One function, so -// the two verbs cannot differ (novox/hq ADR 0450 §2, issue 1324). -func deliveryPlanDocument(m orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge, policy policyOf) map[string]any { - s := pullScope(m.pull(), entries, read, edges) - plan := changePlanOf(m.Repository, m.Base, m.Head, s.Reach, entries, policy) - return map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents, - New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()} -} - -// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner: -// -// delivery plan --repository owner/repo --head [--base main] --paths a,b [--module-dirs …] [--removed …] -// delivery order --members -// delivery check --group --members -// delivery go [--by ] [--why ] -// delivery stop --why [--by ] -// delivery walks [-n 50] [--plan ] -func deliveryCommand(ctx context.Context, args []string) error { - if len(args) == 0 { - return errors.New("delivery order|go|stop|walks") - } - sub, rest := args[0], args[1:] - if sub == "plan" { - // Removed (ADR 0450 §8; issue 1318): refused by name before any flag is read, naming its replacement. - return removedByName("delivery-plan") - } - if sub == "check" { - // Removed (ADR 0503 §3, ADR 0527; issue 1318): the checker judges every head, so the controller asks nothing. - return removedByName("delivery-check") - } - set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError) - membersJSON := set.String("members", "", "a group's members, as JSON") - membersFile := set.String("members-file", "", "a file of a group's members, as JSON: the same as --members, for a list "+ - "too long for an argument (novox/hq issue 1272)") - ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+ - "(novox/hq ADR 0406 §3)") - by := set.String("by", catalogue.DeliverySeat, "who says it") - why := set.String("why", "", "why") - without := set.String("without", "", "merges a stop leaves out, @, comma-separated: the walk's "+ - "other merges are walked again without them (novox/hq ADR 0299)") - limit := set.Int("n", 50, "how many ended walks to answer beside the open ones") - planID := set.String("plan", "", "one walk") - positionals, err := parseAround(set, rest) - if err != nil { - return err - } - var members []orderMember - if *membersFile != "" { - raw, err := os.ReadFile(*membersFile) - if err != nil { - return fmt.Errorf("--members-file: %w", err) - } - *membersJSON = string(raw) - } - if *membersJSON != "" { - if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil { - return fmt.Errorf("the members are not readable: %w", err) - } - // Marked ready by the verb's word for every member it names, besides any member that says so itself. - if *ready { - for i := range members { - members[i].Ready = true - } - } - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - answer := func(v any) error { - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - return enc.Encode(v) - } - switch sub { - case "order": - if len(members) == 0 { - return errors.New("delivery order --members : a group has members") - } - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - return err - } - changes, err := changesOf(ctx, changeSourcesOf(members, entries, reachOfMembers(members, entries, read, edges))) - if err != nil { - return fmt.Errorf("what the members say of their manifests cannot be read: %w", err) - } - return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges, changes)) - case "go": - if len(positionals) != 1 { - return errors.New("delivery go [--by ] [--why ]") - } - var p inventory.Plan - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why)) - return err - }); err != nil { - return err - } - fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository, - short(p.Commit), *by) - return nil - case "stop": - if len(positionals) != 1 || strings.TrimSpace(*why) == "" { - return errors.New("delivery stop --why [--by ] [--without @,…]") - } - var p inventory.Plan - var said stopAnswer - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without), - time.Now().UTC()) - return err - }); err != nil { - return err - } - if *without != "" { - // What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299). - return answer(said) - } - fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+ - "is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers)) - return nil - case "backups-again": - if len(positionals) != 1 || strings.TrimSpace(*why) == "" { - return errors.New("delivery backups-again --why [--by ]") - } - var p inventory.Plan - var again []inventory.PlanBackup - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, again, err = backupsAgain(ctx, inv, positionals[0], *by, strings.TrimSpace(*why)) - return err - }); err != nil { - return err - } - return answer(map[string]any{"plan": p.ID, "asked_again": again}) - case "walks": - var walks []inventory.Plan - if *planID != "" { - p, err := inv.PlanByID(ctx, *planID) - if err != nil { - return err - } - walks = []inventory.Plan{p} - } else { - if walks, err = inv.OpenPlans(ctx); err != nil { - return err - } - // And the batch being assembled (novox/hq ADR 0276): never a walk to link or let go, shown. - batches, err := inv.Batches(ctx) - if err != nil { - return err - } - walks = append(walks, batches...) - recent, err := inv.RecentPlans(ctx, *limit) - if err != nil { - return err - } - for _, p := range recent { - if !slices.ContainsFunc(walks, func(w inventory.Plan) bool { return w.ID == p.ID }) { - walks = append(walks, p) - } - } - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - // Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6). - now := time.Now() - for i := range walks { - walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv)) - } - return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks, - "own-path": sortedKeysOf(ownPathOf(entries))}) - } - return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub) -} - -// appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations -// (novox/hq issue 582); a lookup that fails is a report not read, which leaves the walk's end unknown rather than wrong. -func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup { - return func(node string, send inventory.SentDeclaration, walkSent time.Time) (inventory.AppliedReport, bool) { - r, ok, err := inv.FirstAppliedAfter(ctx, node, send, walkSent) - if err != nil { - fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err) - return inventory.AppliedReport{}, false - } - // And, for each module the send handed to the provisioner, its client's report on that build (novox/hq ADR - // 0362): applied only once both said it, at the later; failed if the client failed. - return inventory.ProvisionedSend(node, send, r, ok, provLookup(ctx)) - } -} - -// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind -// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown -// past ApplySilentAfter keeps its measured phases without its total. -func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error { - recent, err := inv.RecentPlans(ctx, 30) - if err != nil { - return err - } - for _, p := range recent { - if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour { - continue - } - anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID) - if err != nil { - return err - } - if totalKept { - continue - } - ph := p.WalkPhases(now, appliedFrom(ctx, inv)) - if ph != nil && ph.End == nil && anyKept { - continue // kept without its end; kept again only once its end is known - } - if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) { - continue - } - class := ph.Class - if class == "" { - class = "unclassed" - } - for _, x := range ph.Phases { - if x.State != inventory.PhaseMeasured || x.Start == nil { - continue - } - if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase, - Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start, - Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil { - return err - } - } - if ph.End != nil && ph.From != nil { - if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase, - Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From, - Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil { - return err - } - } - } - return nil -} - -// theGraph is what the planner reads. -func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository, - []inventory.Edge, error) { - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil, nil, nil, err - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return nil, nil, nil, err - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return nil, nil, nil, err - } - return entries, read, edges, nil -} - -// appendNew appends what is not there yet. -func appendNew(to []string, items ...string) []string { - for _, i := range items { - if !slices.Contains(to, i) { - to = append(to, i) - } - } - return to -} - -// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the -// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could -// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by -// comparison. Records arriving out of order are told apart by the plan's revision. -// -// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and, -// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what -// the walk recorded. -// -// The plan the announcement is made from is a copy that shares nothing with the walk, taken before the goroutine -// starts: the walk goes on writing its steps' maps and pointers while the announcement is marshalled and its -// touches read (novox/hq issue 1260), and it says the plan as it was when this was called. -func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) { - kept, err := copyOfPlan(p) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err) - return - } - go publishPlanMoved(ctx, bus, facts, kept) -} - -// copyOfPlan is the plan with nothing shared with the original: its JSON round trip, which is all of it but what -// the plan never keeps (WalkSaid), carried over. -func copyOfPlan(p inventory.Plan) (inventory.Plan, error) { - raw, err := json.Marshal(p) - if err != nil { - return inventory.Plan{}, err - } - var kept inventory.Plan - if err := json.Unmarshal(raw, &kept); err != nil { - return inventory.Plan{}, err - } - kept.WalkSaid = p.WalkSaid - return kept, nil -} - -func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) { - // Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are - // read by whoever asks for the walk (`delivery walks`). - p.Phases = p.WalkPhases(time.Now(), nil) - body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(ctx, facts, p)}) - if err != nil { - return - } - stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer stop() - if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, body); err != nil { - fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err) - } -} - -// sayingOnTheBus runs a command that keeps walks or says verdicts with the bus to say them on: the serving -// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept -// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command -// still runs; what it did is found by comparison. -func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error { - if checkEvents != nil { - return f() - } - ran := false - err := onTheBus(func(conn *nats.Conn) error { - js, err := conn.JetStream() - if err != nil { - return err - } - bus := link.OverNATS{Conn: conn, JS: js} - was := inventory.PlanSaved - checkEvents = bus - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { publishPlanMoved(ctx, bus, facts, p) } - defer func() { checkEvents, inventory.PlanSaved = nil, was }() - ran = true - return f() - }) - if !ran { - fmt.Fprintf(os.Stderr, "the bus cannot be reached (%v): what this does is not said, and is found by comparison\n", err) - return f() - } - return err -} - -// sayingWhatAVerbKept is PlanSaved for a process that is not the serving controller: each walk the verb keeps is -// said as `plan-moved` before the save returns, with the facts of the inventory that kept it, on the bus the verb -// reaches. It reaches the bus on the first save and never for a verb that keeps no walk. When the bus cannot be -// reached, a save that started or ended a walk says so on the verb's error output: its mark is kept and no later -// save says it, so the router holds what the walk touched until its bound, or not at all for a start. done lets -// the bus go and takes the hook back. -func sayingWhatAVerbKept(ctx context.Context) (said func(*inventory.Inventory, inventory.Plan), done func()) { - var ( - mu sync.Mutex - reached bool - bus link.Bus - closing func() - failed error - ) - said = func(inv *inventory.Inventory, p inventory.Plan) { - mu.Lock() - defer mu.Unlock() - if !reached { - reached = true - if handActConn != nil { - bus, failed = overNATS(handActConn) - } else if js, err := aBus(); err != nil { - failed = err - } else if bus, failed = overNATS(js.Conn()); failed == nil { - closing = js.Close - } else { - js.Close() - } - } - if bus == nil { - switch p.WalkSaid { - case inventory.WalkStarted: - fmt.Fprintf(os.Stderr, "%s: its start is kept and could NOT be said as plan-moved (%v): the router "+ - "does not hold what it touches\n", p.ID, failed) - case inventory.WalkEnded: - fmt.Fprintf(os.Stderr, "%s: its end is kept and could NOT be said as plan-moved (%v): the router holds "+ - "what it touched until its bound\n", p.ID, failed) - } - return - } - publishPlanMoved(ctx, bus, inv, p) - } - done = func() { - mu.Lock() - defer mu.Unlock() - inventory.PlanSaved = nil - if closing != nil { - closing() - } - } - return said, done -} - -// overNATS is the bus a connection carries, with its JetStream. -func overNATS(conn *nats.Conn) (link.Bus, error) { - js, err := conn.JetStream() - if err != nil { - return nil, err - } - return link.OverNATS{Conn: conn, JS: js}, nil -} diff --git a/cmd/mesh-controller/delivery_conditions.go b/cmd/mesh-controller/delivery_conditions.go deleted file mode 100644 index c5de9415..00000000 --- a/cmd/mesh-controller/delivery_conditions.go +++ /dev/null @@ -1,262 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table -// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the -// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery..stalled`; healer -// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only -// the next observation says whether it worked (ADR 0231). - -// probeDeliveriesID is the probe that reads the delivery's owner. -const probeDeliveriesID = "D14" - -// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope. -const kindDeliveryStalled = "stalled" - -// deliveryOwnerWithin is how long the owner is given to answer. -var deliveryOwnerWithin = 10 * time.Second - -// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it. -type stalledLine struct { - ID string `json:"id"` - // Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347). - Number int `json:"number,omitempty"` - State string `json:"state"` - // Walk is the walk a line is about: a waiting walk no live delivery holds, or, asked with `walks`, each walk - // waiting for the owner's word and what holds it (novox/hq issues 362, 529). - Walk string `json:"walk,omitempty"` - // Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each: - // "mesh/check pending since ", "mesh/check never set" (novox/hq issue 438). - Waiting []string `json:"waiting,omitempty"` - // Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq - // issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before - // says none, and Waiting is said as it reads. - Checks []waitingCheck `json:"checks,omitempty"` - // Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held - // (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no - // delivery, and from a mesh-delivery that does not say it yet; the words are never read for it. - Held string `json:"held,omitempty"` - For string `json:"for"` - Bound string `json:"bound"` - H2 string `json:"h2"` - Says string `json:"says"` - // Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty - // or anything else is a warning, as before. - Severity string `json:"severity,omitempty"` - // Needs is "new-head" for a delivery whose cure is its author's new head (novox/hq ADR 0505 §5), Author that author - // as the operator reads it ("the author of feat/x", with the session that pushed it when its pull request names one) - // and Branch its branch. A mesh-delivery from before says none, and the words are as before. - Needs string `json:"needs,omitempty"` - Author string `json:"author,omitempty"` - Branch string `json:"branch,omitempty"` -} - -// needsNewHead is what a stalled line needs when its cure is its author's new head (ADR 0505 §5). -const needsNewHead = "new-head" - -// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise. -func (l stalledLine) severity() conditions.Severity { - if l.Severity == string(conditions.Urgent) { - return conditions.Urgent - } - return conditions.Warning -} - -// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state — -// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say. -type waitingCheck struct { - Context string `json:"context"` - State string `json:"state"` - Since string `json:"since,omitempty"` -} - -// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces. -var wordsNow = time.Now - -// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its -// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads. -var wordsZone = func() *time.Location { return time.Local } - -// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's. -func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") } - -// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted; -// a seam a test replaces. The holder's own refusal is an error naming it. -var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { - granted := false - for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner { - granted = granted || v.Verb == verb - } - if !granted { - return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb) - } - if conn == nil { - return nil, errors.New("this controller is not on the bus") - } - answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin) - if err != nil { - return nil, err - } - if answer.Error != "" { - return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error) - } - return unwrapToolResult(answer.Result), nil -} - -// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the -// protocol's content list holding it as text. -func unwrapToolResult(raw json.RawMessage) json.RawMessage { - var wrapped struct { - Content []struct { - Text string `json:"text"` - } `json:"content"` - } - if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) { - return json.RawMessage(wrapped.Content[0].Text) - } - return raw -} - -// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record, -// or none answers (D3 says that one). -func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) { - if !held { - return nil, nil - } - raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{}) - if errors.Is(err, link.ErrNothingServes) { - return nil, nil // the holder not running is D3's holder-silent, said once there - } - if err != nil { - return nil, err - } - var lines []stalledLine - if err := json.Unmarshal(raw, &lines); err != nil { - return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err) - } - return lines, nil -} - -// walkWaits are, by walk, the lines mesh-delivery says of a walk waiting for its word: what holds it. -type walkWaits map[string][]stalledLine - -// walksWaiting asks the delivery's owner what holds each walk waiting for its word (novox/hq issue 529): its -// `stalled` asked with `walks`, so a line of `plans` names a held delivery, its reason and the verb that releases -// it, instead of pointing at a hand start. Nil when no holder is on record or none answers: the line then reads as -// before. An owner from before this answers its stalled lines, of which only those naming a walk are kept. -func walksWaiting(ctx context.Context, conn *nats.Conn, held bool) walkWaits { - if !held { - return nil - } - raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{"walks": true}) - if err != nil { - return nil - } - var lines []stalledLine - if json.Unmarshal(raw, &lines) != nil { - return nil - } - out := walkWaits{} - for _, l := range lines { - if l.Walk != "" { - out[l.Walk] = append(out[l.Walk], l) - } - } - return out -} - -// stalledObservations are the conditions of what is stalled. -func stalledObservations(lines []stalledLine) []conditions.Observation { - out := make([]conditions.Observation, 0, len(lines)) - for _, l := range lines { - o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled, - Severity: l.severity(), - Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+ - "`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID), - Said: fmt.Sprintf("%s for %s", l.State, l.For)} - if l.operatorsOnly() { - o.Resolver = conditions.ResolverOperator - } - o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o) - out = append(out, o) - } - return out -} - -// probeDeliveries is D14: every delivery held past its state's bound is said. -func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - entries, err := d.open.inventory.Catalogued(ctx) - if err != nil { - return nil, err - } - var conn *nats.Conn - if d.js != nil { - conn = d.js.Conn() - } - lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries)) - if err != nil { - return nil, err - } - return stalledObservations(lines), nil -} - -// --- H2, for a delivery --------------------------------------------------------------------------------- - -// connOf is the bus a healer asks over. -func (h *healing) connOf() *nats.Conn { - if h.js == nil { - return nil - } - return h.js.Conn() -} - -// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table -// lets H2 take; never one whose state is the operator's. -func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) { - lines, err := deliveriesStalled(ctx, h.connOf(), true) - if err != nil { - return "", false, "", err - } - for _, l := range lines { - if l.ID != c.Subject.ID { - continue - } - if l.operatorsOnly() { - return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil - } - return c.Key, true, "", nil - } - return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil -} - -// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the -// transition its table names. A refusal is no repair, said; the next observation says whether it worked. -func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) { - raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key}) - if err != nil { - if errors.Is(err, link.ErrNothingServes) { - return "", "", err - } - return "closed nothing", oneLine(err.Error()), nil - } - var said string - if json.Unmarshal(raw, &said) != nil { - said = string(raw) - } - return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil -} diff --git a/cmd/mesh-controller/delivery_conditions_test.go b/cmd/mesh-controller/delivery_conditions_test.go deleted file mode 100644 index c9533fac..00000000 --- a/cmd/mesh-controller/delivery_conditions_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "slices" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's -// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`. - -// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records -// what close was asked. -func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string { - t.Helper() - var closed []string - was := askDeliveryOwner - askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { - switch verb { - case "stalled": - raw, _ := json.Marshal(lines) - return raw, nil - case "close": - closed = append(closed, args["id"].(string)) - if closeErr != nil { - return nil, closeErr - } - return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil - } - t.Fatalf("asked the owner %s", verb) - return nil, nil - } - t.Cleanup(func() { askDeliveryOwner = was }) - return &closed -} - -func holdTheDeliverySeat(t *testing.T, open *stores) { - t.Helper() - m := catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}} - if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } -} - -var twoStalled = []stalledLine{ - {ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s", - H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"}, - {ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s", - H2: "none: the state is the operator's", Says: "it waits for the operator"}, -} - -func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - ownerAnswers(t, twoStalled, nil) - d := &doctor{open: open} - got, err := probeDeliveries(ctx, d) - if err != nil || len(got) != 0 { - t.Fatalf("with no holder on record D14 said %+v %v", got, err) - } - holdTheDeliverySeat(t, open) - got, err = probeDeliveries(ctx, d) - if err != nil || len(got) != 2 { - t.Fatalf("D14 said %+v %v", got, err) - } - if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning || - got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") { - t.Fatalf("the first is %+v", got[0]) - } - if got[1].Resolver != conditions.ResolverOperator { - t.Fatalf("a held delivery is not the operator's: %+v", got[1]) - } - // The holder not running is D3's to say, not D14's. - was := askDeliveryOwner - askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) { - return nil, link.ErrNothingServes - } - defer func() { askDeliveryOwner = was }() - if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 { - t.Fatalf("an owner that is down was said by D14: %+v %v", got, err) - } -} - -// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a -// refusal is no repair. -func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - closed := ownerAnswers(t, twoStalled, nil) - for _, o := range stalledObservations(twoStalled) { - o.Source = probeDeliveriesID - if _, err := conditionsFrom.Observe(ctx, o); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) - if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) { - t.Fatalf("close was asked for %v", *closed) - } - acts := told.said() - if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" { - t.Fatalf("said %+v", acts) - } - // A refusal: closed nothing, said so. - c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled") - ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering")) - act, said, err := repairDelivery(ctx, h, c) - if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") { - t.Fatalf("a refused close reads %q %q %v", act, said, err) - } -} - -// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else -// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read. -func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) { - granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"}) - if err != nil { - t.Fatal(err) - } - // And release and stop, which the operator's warrant chooses (novox/hq ADR 0259). - for _, verb := range []string{"stalled", "close", "release", "stop"} { - if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) { - t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb) - } - } - if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") { - t.Fatalf("a verb the grant does not name was asked: %v", err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - if _, err := deliveriesStalled(t.Context(), conn, true); err != nil { - t.Fatalf("an owner not running is D3's, not an error: %v", err) - } - lines, _ := json.Marshal(twoStalled) - wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}}) - sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) { - body, _ := json.Marshal(link.Answer{Result: wrapped}) - _ = m.Respond(body) - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() - got, err := deliveriesStalled(ctx, conn, true) - if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" { - t.Fatalf("over the bus: %+v %v", got, err) - } -} diff --git a/cmd/mesh-controller/delivery_edges.go b/cmd/mesh-controller/delivery_edges.go deleted file mode 100644 index 9228c75b..00000000 --- a/cmd/mesh-controller/delivery_edges.go +++ /dev/null @@ -1,212 +0,0 @@ -package main - -import ( - "fmt" - "slices" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The order of a plan's modules is read from the module graph and from what the change says of the manifests -// (novox/hq ADR 0419), per module and never per repository: no `after:` line is read, and no member of a group -// is ordered against another. Pure, so a test needs no store. - -// The rules that draw an edge of a plan's order, in the words the plan shows. -const ( - ruleVerbBeforeCaller = "verb before caller" - ruleVersionSkew = "version skew" - ruleEngineFirst = "engine before controller" -) - -// switchField is the line of a pull request's description that names release 6's switch (novox/hq ADR 0539 §11): -// "Release-switch: 6". A plan whose merges include a pull request with it puts the node-engine and the controller in -// one tier. Read by switchNamed, from the description the forge announced with the merge. -const switchField = "Release-switch:" - -// The two modules the rollout rules name. -const ( - theController = "mesh-controller" - theEngine = "mesh-host" -) - -// moduleChange is what a change says of one module's manifest: the seat verbs (".") it adds to what the -// module serves, the seat verbs it adds to what the module invokes, and whether the manifest changes at all. -type moduleChange struct { - Serves []string - Invokes []string - Manifest bool -} - -// planOrder is the edges the rules draw among a plan's modules, and the cycle they make when they make one. -type planOrder struct { - Edges []inventory.Edge - // Cycle is every edge of a cycle among the order's edges and the graph's, the module graph's included. - Cycle []inventory.Edge - // Together is, when the switch is named and the plan moves the controller, the line its tier carries (ADR 0539 - // §11): the record and each edge it overruled (ADR 0419 clause 5). Empty otherwise. - Together string -} - -// switchNamed says whether a pull request's description names release 6's switch: a line that is exactly the field -// with the value 6. -func switchNamed(body string) bool { - for _, line := range strings.Split(body, "\n") { - if rest, ok := strings.CutPrefix(strings.TrimSpace(line), switchField); ok && strings.TrimSpace(rest) == "6" { - return true - } - } - return false -} - -// CycleSaid names every module and edge of the cycle, for a check to fail with (ADR 0419 clause 6). -func (o planOrder) CycleSaid() string { - if len(o.Cycle) == 0 { - return "" - } - var parts []string - for _, e := range o.Cycle { - parts = append(parts, edgeSaid(e)) - } - return "the modules of the plan are ordered in a cycle: " + strings.Join(parts, "; ") -} - -// edgeSaid is an edge as a person reads it: the module after, the module before, and why. -func edgeSaid(e inventory.Edge) string { - why := e.Why - if why == "" { - why = e.Kind - } - return fmt.Sprintf("%s after %s (%s)", e.From, e.To, why) -} - -// orders is whether an edge of the module graph orders a plan's tiers (tiersOf). -func orders(e inventory.Edge) bool { - return e.Kind != inventory.EdgePackages && e.Kind != inventory.EdgeBuiltBy && e.From != e.To -} - -// orderEdges draws the edges of a plan's order among its moved modules (ADR 0419 clause 3, 4): a module whose change -// adds a verb to a seat goes before one whose change adds a call of that verb; the controller, when the plan moves -// it, goes before every module whose manifest the plan changes, the node-engine excepted; and the node-engine goes -// before the controller by default, when the plan moves both and no edge orders them either way, transitively. An -// edge is "From after To". graph is the module graph, whose edges order too and are never drawn again. -// switchNamed is release 6's switch (ADR 0539 §11): the version-skew edge and the engine-first default are not drawn -// and the plan says it overruled them (Together); the verb edges and the graph still order. -func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge, switchNamed bool) planOrder { - var out planOrder - sorted := append([]string(nil), moved...) - sort.Strings(sorted) - // The switch acts only on a plan that moves both the engine and the controller. - switchNamed = switchNamed && slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) - for _, server := range sorted { - for _, caller := range sorted { - if server == caller { - continue - } - for _, verb := range changes[server].Serves { - if slices.Contains(changes[caller].Invokes, verb) { - out.Edges = append(out.Edges, inventory.Edge{From: caller, To: server, Kind: edgeChangeOrder, - Why: ruleVerbBeforeCaller + ": " + verb}) - } - } - } - } - var overruled []string - if slices.Contains(sorted, theController) { - for _, m := range sorted { - if m != theController && m != theEngine && changes[m].Manifest { - if switchNamed { - overruled = append(overruled, ruleVersionSkew+" ("+m+" after "+theController+")") - continue - } - out.Edges = append(out.Edges, inventory.Edge{From: m, To: theController, Kind: edgeChangeOrder, - Why: ruleVersionSkew}) - } - } - } - if switchNamed { - overruled = append(overruled, ruleEngineFirst) - } - if switchNamed { - out.Together = switchSaid(overruled) - } else if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) { - all := orderingEdges(graph, out.Edges) - switch { - case pathBetween(all, theController, theEngine) != nil: - // An edge already puts the controller after the engine: the default says the same. - case pathBetween(all, theEngine, theController) != nil: - // An edge puts the engine after the controller: it overrules the default, and says so. - path := pathBetween(all, theEngine, theController) - for i := range out.Edges { - if out.Edges[i] == path[0] { - out.Edges[i].Why += " (overrules " + ruleEngineFirst + ")" - break - } - } - default: - out.Edges = append(out.Edges, inventory.Edge{From: theController, To: theEngine, Kind: edgeChangeOrder, - Why: ruleEngineFirst}) - } - } - all := orderingEdges(graph, out.Edges) - for _, e := range out.Edges { - if back := pathBetween(all, e.To, e.From); back != nil { - out.Cycle = append(out.Cycle, e) - out.Cycle = append(out.Cycle, back...) - break - } - } - return out -} - -// switchSaid is the line of a tier the switch shapes: the record that decided it and each edge it overruled. -func switchSaid(overruled []string) string { - said := "release 6's switch (ADR 0539 §11)" - if len(overruled) > 0 { - said += " overrules " + strings.Join(overruled, ", ") + " (ADR 0419 clause 5)" - } - return said -} - -// orderingEdges is the graph's edges that order and the edges drawn, together. -func orderingEdges(graph, drawn []inventory.Edge) []inventory.Edge { - var out []inventory.Edge - for _, e := range graph { - if orders(e) { - out = append(out, e) - } - } - return append(out, drawn...) -} - -// pathBetween is the edges that lead from a module to another along "depends on" (From to To), the shortest, or nil -// when the first does not depend on the second. -func pathBetween(edges []inventory.Edge, from, to string) []inventory.Edge { - if from == to { - return nil - } - via := map[string]inventory.Edge{} - seen := map[string]bool{from: true} - queue := []string{from} - for len(queue) > 0 { - at := queue[0] - queue = queue[1:] - for _, e := range edges { - if e.From != at || seen[e.To] { - continue - } - seen[e.To] = true - via[e.To] = e - if e.To == to { - var path []inventory.Edge - for m := to; m != from; m = via[m].From { - path = append([]inventory.Edge{via[m]}, path...) - } - return path - } - queue = append(queue, e.To) - } - } - return nil -} diff --git a/cmd/mesh-controller/delivery_edges_test.go b/cmd/mesh-controller/delivery_edges_test.go deleted file mode 100644 index aa71bcfc..00000000 --- a/cmd/mesh-controller/delivery_edges_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package main - -import ( - "context" - "slices" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The order of a plan's modules is read from the module graph and the manifests' change alone (novox/hq ADR 0419): -// no `after:` line, no member of a group. Pure, like orderOf was. - -func hasEdge(edges []inventory.Edge, from, to string) (inventory.Edge, bool) { - for _, e := range edges { - if e.From == from && e.To == to { - return e, true - } - } - return inventory.Edge{}, false -} - -// A module whose change adds a verb to a seat goes before the module whose change adds a call of it: the caller -// depends on the server, the plan puts them in two tiers, and the edge says why. -func TestAVerbIsServedBeforeItsCallerIsSent(t *testing.T) { - changes := map[string]moduleChange{ - "server": {Serves: []string{"mesh-delivery.groups"}, Manifest: true}, - "caller": {Invokes: []string{"mesh-delivery.groups"}, Manifest: true}, - } - got := orderEdges([]string{"caller", "server"}, changes, nil, false) - e, ok := hasEdge(got.Edges, "caller", "server") - if !ok || !strings.Contains(e.Why, "verb before caller") || !strings.Contains(e.Why, "mesh-delivery.groups") { - t.Fatalf("the caller does not follow the server by the verb: %+v", got.Edges) - } - if len(got.Cycle) != 0 { - t.Fatalf("a cycle: %+v", got.Cycle) - } - if tiers := tiersOf([]string{"caller", "server"}, got.Edges); len(tiers) != 2 || tiers[0][0] != "server" { - t.Fatalf("tiers %v, want the server first", tiers) - } -} - -// Edges of one rule both ways are a cycle: every module and edge is named. -func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) { - changes := map[string]moduleChange{ - "x": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}}, - "y": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}}, - } - got := orderEdges([]string{"x", "y"}, changes, nil, false) - if len(got.Cycle) < 2 { - t.Fatalf("no cycle found: %+v", got) - } - said := got.CycleSaid() - for _, want := range []string{"x", "y", "s.a", "s.b"} { - if !strings.Contains(said, want) { - t.Fatalf("the cycle is said as %q, without %s", said, want) - } - } -} - -// The node-engine goes before the controller by default, and says so; an edge of the manifests the other way -// overrules it, and the edge that did is said. -func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T) { - got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil, false) - e, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host") - if !ok || e.Why != "engine before controller" { - t.Fatalf("the engine does not go first: %+v", got.Edges) - } - changes := map[string]moduleChange{ - "mesh-host": {Serves: []string{"s.v"}}, - "mesh-controller": {Invokes: []string{"s.v"}}, - } - got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil, false) - if _, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host"); !ok || len(got.Edges) != 1 || - !strings.Contains(got.Edges[0].Why, "verb before caller") { - t.Fatalf("the verb edge does not place the controller after the engine: %+v", got.Edges) - } - // The other way: the engine calls what the controller brings. - changes = map[string]moduleChange{ - "mesh-controller": {Serves: []string{"s.v"}}, - "mesh-host": {Invokes: []string{"s.v"}}, - } - got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil, false) - e, ok = hasEdge(got.Edges, "mesh-host", "mesh-controller") - if !ok || len(got.Edges) != 1 || !strings.Contains(e.Why, "overrules engine before controller") { - t.Fatalf("the verb edge does not say it overruled the default: %+v", got.Edges) - } -} - -// The controller goes before every module whose manifest the plan changes (version skew), the node-engine excepted. -func TestTheControllerGoesBeforeEveryChangedManifest(t *testing.T) { - changes := map[string]moduleChange{"app": {Manifest: true}, "other": {}} - got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil, false) - if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" { - t.Fatalf("the changed manifest does not follow the controller: %+v", got.Edges) - } - if _, ok := hasEdge(got.Edges, "other", "mesh-controller"); ok { - t.Fatalf("a module with an unchanged manifest was ordered: %+v", got.Edges) - } - if got := orderEdges([]string{"app", "other"}, changes, nil, false); len(got.Edges) != 0 { - t.Fatalf("version skew ordered a plan that does not move the controller: %+v", got.Edges) - } -} - -// The order is per module, not per repository: of two modules of one repository only the one with the edge moves. -func TestTheOrderIsPerModuleNotPerRepository(t *testing.T) { - changes := map[string]moduleChange{ - "a": {Serves: []string{"s.v"}}, - "c": {Invokes: []string{"s.v"}}, - } - got := orderEdges([]string{"a", "b", "c"}, changes, nil, false) - tiers := tiersOf([]string{"a", "b", "c"}, got.Edges) - if len(tiers) != 2 || strings.Join(tiers[0], ",") != "a,b" || strings.Join(tiers[1], ",") != "c" { - t.Fatalf("tiers %v, want [a b] then [c]", tiers) - } -} - -// The edges a plan's order was drawn with are kept on the plan, and its gates read them: a later tier waits for the -// earlier to be applied although the catalogue's graph holds no such edge (ADR 0419, the lead's settlement Q4). -func TestAPlansOwnOrderGatesItsTiersWhateverTheStoreGraphHolds(t *testing.T) { - p := inventory.Plan{Tiers: [][]string{{"server"}, {"caller"}}, Tier: 0, - Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "caller", To: "server", Kind: edgeChangeOrder, - Why: ruleVerbBeforeCaller + ": s.v"}}}} - if got := gates(p, nil, func(string) bool { return true }); len(got) != 1 || got[0] != "server" { - t.Fatalf("the caller's tier waits for %v; want the server", got) - } - p.Delivery.Order = nil - if got := gates(p, nil, func(string) bool { return true }); len(got) != 0 { - t.Fatalf("a plan with no order waits for %v", got) - } -} - -// A verb an earlier merge of the same repository in the batch added is added by the batch: the change is compared with -// what the repository held before the batch, not with the first parent of its latest merge. -func TestAVerbAnEarlierMergeOfTheBatchAddedDrawsItsEdge(t *testing.T) { - was, wasParent := manifestAt, parentOf - t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) - manifestsReadMu.Lock() - clear(manifestsRead) - manifestsReadMu.Unlock() - const first, second, before = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "0000000000000000000000000000000000000000" - parentOf = func(_ context.Context, _, _, commit string) (string, error) { - if commit == first { - return before, nil - } - return first, nil // the latest merge's parent holds the verb already - } - manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) { - m := catalogue.Manifest{Module: "server"} - if ref != before { - m.DefinesSeats = []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "v"}}}} - } - return m, true, nil - } - src := changeSource{Owner: "novox", Repo: "r", New: second, Paths: []string{"module.json"}} - if got, err := changesOf(context.Background(), []changeSource{src}); err != nil || len(got["server"].Serves) != 0 { - t.Fatalf("against the first parent of the latest merge: %+v %v", got, err) - } - src.Since = first - got, err := changesOf(context.Background(), []changeSource{src}) - if err != nil || len(got["server"].Serves) != 1 || got["server"].Serves[0] != "s.v" { - t.Fatalf("against what the batch found: %+v %v", got, err) - } -} - -// A change that names only a beginning of its files is read as changing the manifests of the modules it moves. -func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T) { - entries := []inventory.Entry{ - {Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}}, - {Manifest: catalogue.Manifest{Module: "b"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/b"}}, - {Manifest: catalogue.Manifest{Module: "c"}, Source: inventory.Source{Repository: "novox/other", Path: "modules/c"}}, - } - got := truncatedPaths(entries, "novox", "mesh-catalog", []string{"a", "c"}) - if len(got) != 1 || got[0] != "modules/a/module.json" { - t.Fatalf("read as changed: %v", got) - } - members := []orderMember{{ID: "m", Repository: "novox/mesh-catalog", Head: "h", PathsTruncated: true, Paths: []string{"x"}}} - src := changeSourcesOf(members, entries, map[string]orderReach{"m": {Moved: []string{"a"}}}) - if len(src) != 1 || src[0].Truncated || len(src[0].Paths) != 1 || src[0].Paths[0] != "modules/a/module.json" { - t.Fatalf("the member's change is read as %+v", src) - } -} - -// 1942/AC1: release 6's switch puts the node-engine and the controller in one tier; without it the controller's -// tier comes first as today (ADR 0539 §11). -func TestTheSwitchPutsTheEngineAndTheControllerInOneTier(t *testing.T) { - moved := []string{"mesh-controller", "mesh-host", "app"} - changes := map[string]moduleChange{"mesh-host": {Manifest: true}, "app": {Manifest: true}} - with := orderEdges(moved, changes, nil, true) - if len(with.Edges) != 0 || len(with.Cycle) != 0 { - t.Fatalf("the switch still draws edges: %+v", with) - } - if tiers := tiersOf(moved, with.Edges); len(tiers) != 1 || len(tiers[0]) != 3 { - t.Fatalf("tiers %v, want one tier", tiers) - } - without := orderEdges(moved, changes, nil, false) - tiers := tiersOf(moved, without.Edges) - if len(tiers) < 2 || slices.Contains(tiers[0], "mesh-controller") && slices.Contains(tiers[0], "mesh-host") { - t.Fatalf("without the switch the tiers are %v, want the engine's tier before the controller's", tiers) - } - if without.Together != "" { - t.Fatalf("a plan without the switch carries its line: %q", without.Together) - } - // A verb edge between them still orders them. - verb := map[string]moduleChange{"mesh-host": {Serves: []string{"s.v"}}, "mesh-controller": {Invokes: []string{"s.v"}}} - if got := orderEdges([]string{"mesh-controller", "mesh-host"}, verb, nil, true); len(got.Edges) != 1 { - t.Fatalf("the switch dropped a verb edge: %+v", got.Edges) - } -} - -// 1942/AC2: the shared tier's line names the record and each edge overruled. -func TestTheSwitchSaysTheEdgesItOverruled(t *testing.T) { - changes := map[string]moduleChange{"app": {Manifest: true}} - got := orderEdges([]string{"mesh-controller", "mesh-host", "app"}, changes, nil, true) - for _, want := range []string{"ADR 0539 §11", "engine before controller", "version skew (app after mesh-controller)", "ADR 0419"} { - if !strings.Contains(got.Together, want) { - t.Fatalf("the line %q lacks %q", got.Together, want) - } - } - plan := inventory.Plan{Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "mesh-host": {}}} - placedTogether(&plan, got.Together) - if plan.Modules["mesh-controller"].Placed != got.Together || plan.Modules["mesh-host"].Placed != got.Together { - t.Fatalf("the tier does not carry the line: %+v %+v", plan.Modules["mesh-controller"], plan.Modules["mesh-host"]) - } -} - -// The switch is named by a line of a pull request's description. -func TestTheSwitchIsNamedByAFieldOfTheDescription(t *testing.T) { - if !switchNamed("Fixes 1\n\nRelease-switch: 6\n") || switchNamed("Release-switch:\n") || switchNamed("Release-switch: 7") || switchNamed("about the Release-switch: 6") { - t.Fatal("the field is not read as a line with a value") - } -} - -// 1942/AC3: the switch acts only on a plan that moves the engine and the controller: without mesh-host the -// version-skew edge stays, the line present or not. -func TestTheSwitchWithoutTheEngineKeepsVersionSkew(t *testing.T) { - changes := map[string]moduleChange{"app": {Manifest: true}} - got := orderEdges([]string{"mesh-controller", "app"}, changes, nil, true) - if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" || got.Together != "" { - t.Fatalf("version skew lost without the engine: %+v", got) - } -} diff --git a/cmd/mesh-controller/delivery_integration_test.go b/cmd/mesh-controller/delivery_integration_test.go deleted file mode 100644 index fa0c04cf..00000000 --- a/cmd/mesh-controller/delivery_integration_test.go +++ /dev/null @@ -1,216 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - withConditionsInMemory(t) - for _, name := range []string{"app", "mesh-delivery"} { - m := catalogue.Manifest{Module: name, Version: "1"} - if name == "mesh-delivery" { - m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}} - } - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - merge := func(commit string, paths ...string) inventory.Plan { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths, - ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || recent[0].Commit != commit { - t.Fatalf("no plan for %s: %v %v", commit, recent, err) - } - return recent[0] - } - - // finish ends a walk as done, so the next merge has nothing of it to take over. - finish := func(id string) { - t.Helper() - w, err := inv.PlanByID(ctx, id) - if err != nil { - t.Fatal(err) - } - w.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - } - - // heardOf has this mesh's own view of the plans take in the plan as the bucket holds it. A verb of the - // controller (deliveryCommand, plansCommand) opens an inventory of its own and writes the plan there; this - // mesh's inventory learns of the write from the bucket's watch, a moment later, and a loaded machine makes the - // moment long enough for advanceHeld to read the plan as it was, still waiting for its word, and ask nothing - // (issue 1399). Reading the plan from the bucket puts it into the view at once, with no wait to bound. - heardOf := func(id string) { - t.Helper() - if _, err := inv.PlanByID(ctx, id); err != nil { - t.Fatal(err) - } - } - - // No holder on record: the merge starts its walk, as before. - // A cut takes its first tier from the builder's keys and a step of the walk records the take (the walk asks no - // build), so each walk that starts is stepped once before its takes are counted. - p := merge("c1aaaaaaaa", "modules/app/index.ts") - advanceHeld(ctx, open) - if p.Waiting() || len(*asked) != 1 { - t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked) - } - - // The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk - // is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut. - finish(p.ID) - if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - p = merge("c2bbbbbbbb", "modules/app/index.ts") - if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) { - t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note) - } - advanceHeld(ctx, open) - if len(*asked) != 1 { - t.Fatalf("a waiting walk was advanced into asking: %v", *asked) - } - if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") { - t.Errorf("a waiting walk reads %q", line) - } - - // The delivery's word starts it. - if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil { - t.Fatal("a walk was let go twice") - } - heardOf(p.ID) - advanceHeld(ctx, open) - if len(*asked) != 2 { - t.Fatalf("the word did not start the walk: %v", *asked) - } - got, err := inv.PlanByID(ctx, p.ID) - if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" { - t.Fatalf("the word was not kept: %+v %v", got.Delivery, err) - } - - // The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq - // ADR 0276): the merge's batch is cut once it ended. - finish(p.ID) - p = merge("c3cccccccc", "modules/mesh-delivery/main.go") - advanceHeld(ctx, open) - if p.Waiting() || len(*asked) != 3 { - t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked) - } - - // The owner down: a person starts a waiting walk, with why. - finish(p.ID) - p = merge("c4dddddddd", "modules/app/index.ts") - if !p.Waiting() { - t.Fatal("the walk did not wait") - } - if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") { - t.Fatalf("a walk was started by hand without why: %v", err) - } - if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil { - t.Fatal(err) - } - heardOf(p.ID) - advanceHeld(ctx, open) - got, _ = inv.PlanByID(ctx, p.ID) - if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 { - t.Fatalf("a person's word did not start the walk: %+v, asked %v, state %q, tier %d, note %q", got.Delivery, *asked, got.State, got.Tier, got.Note) - } - - // The owner's stop: failed, said as stopped by it. - finish(p.ID) - p = merge("c5eeeeeeee", "modules/app/index.ts") - if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by", - "mesh-delivery for jochen"}); err != nil { - t.Fatal(err) - } - got, _ = inv.PlanByID(ctx, p.ID) - if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" || - !strings.Contains(got.Note, "the operator stopped it") { - t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note) - } - - // Unassigned: the mesh is back on the controller's own path. - if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - asks := len(*asked) - p = merge("c6ffffffff", "modules/app/index.ts") - advanceHeld(ctx, open) - if p.Waiting() || len(*asked) != asks+1 { - t.Fatalf("with the holder gone the walk waited: %v", p.Waiting()) - } -} - -func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7", - Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - var said inventory.Plan - if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil || - said.Delivery.Go == nil { - t.Fatalf("plan-moved said %s (%v)", m.Data, err) - } - case <-time.After(5 * time.Second): - t.Fatal("a walk let go by a verb was not said") - } - if checkEvents != nil || inventory.PlanSaved != nil { - t.Fatal("the command's own bus was left in place") - } -} diff --git a/cmd/mesh-controller/delivery_order_test.go b/cmd/mesh-controller/delivery_order_test.go deleted file mode 100644 index e5776e50..00000000 --- a/cmd/mesh-controller/delivery_order_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "context" - "errors" - "reflect" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// recordingDelivery is a delivery that writes down what was done, in order, and fails where told. -type recordingDelivery struct { - did []string - grantErr error - declareErr error -} - -func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error { - for _, s := range sending { - r.did = append(r.did, "grant "+s.node) - } - return r.grantErr -} - -func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) { - if r.declareErr != nil { - return "", r.declareErr - } - r.did = append(r.did, "declare "+s.node) - return "digest-" + s.node, nil -} - -func ready(names ...string) []readyNode { - var out []readyNode - for _, n := range names { - out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}}) - } - return out -} - -// novox/hq issue 249: the grants that come with a module's new declarations are issued before any -// machine is sent the code that uses them — except the machine holding the bus, whose declaration -// carries the controller's own right to issue them, and goes first. -func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) { - d := &recordingDelivery{} - digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop")) - if err != nil { - t.Fatal(err) - } - want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"} - if !reflect.DeepEqual(d.did, want) { - t.Fatalf("delivered in the order %v, wanted %v", d.did, want) - } - if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" { - t.Fatalf("the digests sent were not answered: %v", digests) - } - - held := &recordingDelivery{} - if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil { - t.Fatal(err) - } - want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"} - if !reflect.DeepEqual(held.did, want) { - t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want) - } -} - -// A grant that cannot be issued holds back the machines it concerns and is an error the caller -// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the -// grant that would let the controller issue memberships is never held behind them. -func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) { - // A failure naming no machine (the buckets): everything but the bus's machine. - d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")} - _, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop")) - if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") || - !strings.Contains(err.Error(), "anchor, laptop not sent") { - t.Fatalf("a failed grant was not said as the send's failure: %v", err) - } - if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) { - t.Fatalf("delivered %v, wanted the bus's machine alone", d.did) - } - - // A membership that failed for one machine: that machine alone. - one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}} - _, err = deliver(t.Context(), one, "", ready("anchor", "laptop")) - if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") { - t.Fatalf("one machine's refused membership was not said: %v", err) - } - if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) { - t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did) - } - - // The announced upgrade that hit it is asked again. - if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) { - t.Fatal("an announcement whose send stopped at its grants is not asked again") - } - if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) { - t.Fatal("any failure is asked again, not only a grant's") - } - - // Nothing to send is nothing granted either. - none := &recordingDelivery{grantErr: errors.New("never asked")} - if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 { - t.Fatalf("an empty send granted or failed: %v %v", none.did, err) - } -} - -// Whether the bus's machine goes first is read from the user list alone, by its digest. -func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) { - list := "users: [a, b]" - if userListBehind(list, digestOf([]byte(list))) { - t.Fatal("the list it was sent reads as behind") - } - if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") { - t.Fatal("a changed or never-sent list reads as current") - } - if userListBehind("", "") { - t.Fatal("a machine sent no list reads as behind") - } - // A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353). - if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) { - t.Fatal("a list nobody could compose reads as current") - } - if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") { - t.Fatal("listBehind does not read the digest as userListBehind does") - } -} - -// The machine holding the bus goes first: its declaration carries the user list the new grants are -// checked against. Among the machines it is moved to the front; not among them it is added only -// when it is behind. -func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) { - for _, c := range []struct { - what string - names []string - holder string - behind bool - want []string - }{ - {"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}}, - {"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}}, - {"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}}, - {"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}}, - {"only it", []string{"novox"}, "novox", false, []string{"novox"}}, - } { - if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) { - t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want) - } - } -} diff --git a/cmd/mesh-controller/delivery_stalled_severity_test.go b/cmd/mesh-controller/delivery_stalled_severity_test.go deleted file mode 100644 index d59eb346..00000000 --- a/cmd/mesh-controller/delivery_stalled_severity_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "encoding/json" - "testing" - - "github.com/novox/mesh-controller/internal/conditions" -) - -func severityRaisedFor(t *testing.T, line string) conditions.Severity { - t.Helper() - var l stalledLine - if err := json.Unmarshal([]byte(line), &l); err != nil { - t.Fatal(err) - } - got := stalledObservations([]stalledLine{l}) - if len(got) != 1 { - t.Fatalf("%d conditions", len(got)) - } - return got[0].Severity -} - -func TestAnUrgentStalledLineRaisesAnUrgentCondition(t *testing.T) { // 1278/AC1 - line := `{"id":"novox/mesh-delivery#5","state":"held","for":"3m0s","bound":"0s","h2":"close","says":"held","severity":"urgent"}` - if got := severityRaisedFor(t, line); got != conditions.Urgent { - t.Errorf("an urgent line raised %q", got) - } -} - -func TestAStalledLineWithoutOrWithAnUnknownSeverityRaisesAWarning(t *testing.T) { // 1278/AC2 - for _, extra := range []string{``, `,"severity":""`, `,"severity":"critical"`, `,"severity":"warning"`} { - line := `{"id":"novox/a#1","state":"unanswered","for":"3m0s","bound":"2m0s","h2":"close","says":"x"` + extra + `}` - if got := severityRaisedFor(t, line); got != conditions.Warning { - t.Errorf("the line %s raised %q", extra, got) - } - } -} diff --git a/cmd/mesh-controller/delivery_test.go b/cmd/mesh-controller/delivery_test.go deleted file mode 100644 index 05cfc7a3..00000000 --- a/cmd/mesh-controller/delivery_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0419: a group's order, from the module graph and the manifests' change alone, the same on every -// reading — what a module stands on before it, the controller before a manifest that needs it, the node-engine -// before the controller, a verb's server before its caller — and no `after:` line read. -func TestAGroupIsOrderedByTheGraphAndTheManifestsChange(t *testing.T) { - members := []orderMember{ - {ID: "cat", Repository: "novox/mesh-catalog"}, - {ID: "ctl", Repository: "novox/mesh-controller"}, - {ID: "host", Repository: "novox/mesh-host"}, - {ID: "base", Repository: "novox/base"}, - {ID: "app", Repository: "novox/app"}, - {ID: "lab", Repository: "novox/lab"}, - } - reach := map[string]orderReach{ - "cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}}, - "ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}}, - "host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}}, - "base": {Moved: []string{"base"}}, - "app": {Moved: []string{"app"}}, - "lab": {Moved: []string{"lab"}}, - } - edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeStandsOn}, - {From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}} - changes := map[string]moduleChange{"gitea": {Manifest: true}, "mesh-controller": {Manifest: true}, - "mesh-host": {Manifest: true}, "lab": {Serves: []string{"seat.v"}}, "app": {Invokes: []string{"seat.v"}}} - got := orderOf(members, reach, edges, changes) - if len(got.Cycle) > 0 { - t.Fatalf("a cycle where there is none: %v", got.Cycle) - } - // By repository among those with nothing before them: app waits for base and lab; gitea's catalogue for the controller. - want := []string{"base", "lab", "app", "host", "ctl", "cat"} - if !reflect.DeepEqual(got.Order, want) { - t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs) - } - why := map[string]string{} - for _, p := range got.Pairs { - why[p.Before+">"+p.After] = p.Why - } - for pair, reason := range map[string]string{"host>ctl": ruleEngineFirst, "ctl>cat": ruleVersionSkew, - "ctl>host": "", "base>app": orderBuiltBy, "lab>app": ruleVerbBeforeCaller + ": seat.v"} { - if why[pair] != reason { - t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason) - } - } - // The same members in another order read the same. - shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]} - if again := orderOf(shuffled, reach, edges, changes); !reflect.DeepEqual(again.Order, want) { - t.Fatalf("another reading ordered %v", again.Order) - } -} - -// A cycle is the one refusal, and it names every module and edge. A description's `after:` line changes nothing. -func TestAGroupInACycleIsNamedAndNothingElseIsRefused(t *testing.T) { - members := []orderMember{ - {ID: "x", Repository: "novox/x", Number: 7}, - {ID: "y", Repository: "novox/y", Number: 9}, - } - reach := map[string]orderReach{"x": {Moved: []string{"mx"}}, "y": {Moved: []string{"my"}}} - changes := map[string]moduleChange{ - "mx": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}}, - "my": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}}, - } - got := orderOf(members, reach, nil, changes) - if !reflect.DeepEqual(got.Cycle, []string{"x", "y"}) || len(got.Order) > 0 { - t.Fatalf("a cycle was ordered: %v, cycle %v", got.Order, got.Cycle) - } - for _, w := range []string{"mx", "my", "s.a", "s.b"} { - if !strings.Contains(got.CycleSaid, w) { - t.Errorf("the cycle does not say %q: %s", w, got.CycleSaid) - } - } - // Without the edges the same members are ordered by repository. - got = orderOf(members, reach, nil, nil) - if !reflect.DeepEqual(got.Order, []string{"x", "y"}) || len(got.Cycle) > 0 || len(got.Pairs) > 0 { - t.Fatalf("members with no edge: %v %v %+v", got.Order, got.Cycle, got.Pairs) - } -} - -// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on -// record, and never one that moves a module on the controller's own path. -func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) { - holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}} - unassigned := holder - unassigned.On = nil - for _, c := range []struct { - entries []inventory.Entry - moved []string - waits bool - }{ - {nil, []string{"gitea"}, false}, - {[]inventory.Entry{unassigned}, []string{"gitea"}, false}, - {[]inventory.Entry{holder}, []string{"gitea", "plex"}, true}, - {[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false}, - {[]inventory.Entry{holder}, []string{"mesh-host"}, false}, - {[]inventory.Entry{holder}, []string{"node-tools"}, false}, - {[]inventory.Entry{holder}, []string{"nats"}, false}, - {[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false}, - } { - d := awaitsFor(c.entries, c.moved) - if (d != nil) != c.waits { - t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0, - c.moved, d != nil, c.waits) - } - if d != nil && d.Awaits != catalogue.DeliverySeat { - t.Errorf("waits for %q", d.Awaits) - } - } -} - -// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the -// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by -// hand when the owner is down, and the owner's stop ends one as stopped. - -// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over. -func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - want []string - }{ - {"deliver", map[string]any{"plan": "plan-1", "why": "its turn"}, - []string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}}, - {"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"}, - []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}}, - {"delivery-walks", map[string]any{}, []string{"delivery", "walks"}}, - {"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}}, - {"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}}, - {"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}}, - } { - got, err := argvFor(c.verb, c.args) - if err != nil || !reflect.DeepEqual(got, c.want) { - t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want) - } - } - if _, err := argvFor("deliver", map[string]any{}); err == nil { - t.Error("deliver without a walk was composed") - } - if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil { - t.Error("a stop without why was composed") - } -} - -// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the -// command's own, so the delivery's owner hears it at once and not only when it reads the walks back. diff --git a/cmd/mesh-controller/dependents_test.go b/cmd/mesh-controller/dependents_test.go deleted file mode 100644 index bc9386a9..00000000 --- a/cmd/mesh-controller/dependents_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else -// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a -// module built on one of those moves as well. -func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) { - entry := func(name string) inventory.Entry { - return inventory.Entry{Manifest: catalogue.Manifest{Module: name}} - } - entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")} - against := map[string][]string{ - "shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"}, - "shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"}, - "postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"}, - "unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"}, - } - got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against) - var names []string - for _, e := range got { - names = append(names, e.Manifest.Module) - } - want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true} - if len(names) != len(want) { - t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names) - } - for _, n := range names { - if !want[n] { - t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names) - } - } - // Nothing moved: nothing follows. - if more := dependentsOf(nil, entries, against); len(more) != 0 { - t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more)) - } -} diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 475d33fb..76a147e1 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -79,7 +79,7 @@ var probeRegistry = []probe{ {ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " + "assigned to a machine is left out of its declaration unsaid", From: "issues 236, 263, 275, 380", Kind: "declaration-refused", - Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1, + Raises: []string{kindLeftOut, kindNeedsOperator}, Phase: 1, run: probeDeclarations}, {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers}, @@ -115,26 +115,11 @@ var probeRegistry = []probe{ kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting}, Phase: 2, run: probeData, Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}}, - // The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by - // the controller, and H2 works it through the owner's `close`. - {ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " + - "`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239", - Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries}, // What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a // node's presence, which the silent check takes. {ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " + "not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156", Kind: kindStuck, Phase: 3, run: probeStuck}, - // A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the - // operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8). - {ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " + - "minutes for mesh-delivery's announcement of the operator's Approve of its head", From: "ADR 0432, issue 1146", - Kind: kindFirstBuildUnasked, Phase: 3, run: probeFirstBuilds}, - // The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget, - // read from mesh-delivery's `times`; a measurement said, never a delivery held. - {ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " + - "its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304", - Kind: kindOverBudget, Phase: 3, run: probeBudgets}, // A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed // connections, which the bus's own module reads. {ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " + @@ -168,30 +153,13 @@ var probeRegistry = []probe{ {ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " + "ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth}, - {ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " + - "build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth}, - {ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus", - From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth}, {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " + "it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeBusHealth}, - // The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its - // condition until a newer build passes; each rollback a witness stands by is said. - {ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " + - "a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack, - Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates}, // The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus. {ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " + "health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8", Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep}, - // A delivery plan held for a backup that failed or stayed silent (ADR 0404 decision 4): one condition per plan. - {ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " + - "answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1, - run: probeBackupHeld}, - // A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it. - {ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " + - "bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1, - run: probeSendKeyRefused}, } // probeVerdict is one probe's outcome in a run. diff --git a/cmd/mesh-controller/durations.go b/cmd/mesh-controller/durations.go deleted file mode 100644 index b03db5ce..00000000 --- a/cmd/mesh-controller/durations.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// What the core's bounds are set from (novox/hq to-be 45 Phase 0). -// -// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog -// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times -// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional -// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving -// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a -// build's outcome — and summarised here per machine, repository or module. - -// durationSummary is one subject's measurements of one kind. -type durationSummary struct { - Kind string `json:"kind"` - Subject string `json:"subject"` - Count int `json:"count"` - Median string `json:"median"` - P90 string `json:"p90"` - Max string `json:"max"` - // Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured - // time: three times the slowest apply (S2), three times the median word interval (S1). - Suggests string `json:"suggests,omitempty"` -} - -func summarise(ds []inventory.Duration) []durationSummary { - type key struct{ kind, subject string } - by := map[key][]time.Duration{} - for _, d := range ds { - k := key{d.Kind, d.Subject} - by[k] = append(by[k], d.Took) - } - var out []durationSummary - for k, took := range by { - slices.Sort(took) - at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] } - s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took), - Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])} - switch k.kind { - case inventory.DurationApply: - s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90" - } - out = append(out, s) - } - sort.Slice(out, func(i, j int) bool { - ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind) - if ki != kj { - return ki < kj - } - return out[i].Subject < out[j].Subject - }) - return out -} - -func round(d time.Duration) string { - switch { - case d < time.Second: - return d.Round(time.Millisecond).String() - case d < time.Minute: - return d.Round(100 * time.Millisecond).String() - default: - return d.Round(time.Second).String() - } -} - -// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data. -func durationsCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("durations", flag.ContinueOnError) - kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", ")) - days := set.Int("days", 14, "how many days back") - asJSON := set.Bool("json", false, "the summary as data") - all := set.Bool("all", false, "every measurement rather than the summary") - if _, err := parseAround(set, args); err != nil { - return err - } - if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) { - return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", ")) - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour)) - if err != nil { - return err - } - if *all { - body, err := json.MarshalIndent(ds, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - return nil - } - summary := summarise(ds) - if *asJSON { - body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - return nil - } - if len(summary) == 0 { - fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+ - "plan-tier and build durations as it hears them\n", *days) - return nil - } - fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days) - fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max") - for _, s := range summary { - fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max) - if s.Suggests != "" { - fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests) - } - } - return nil -} - -// forgettingOldDurations removes what is older than a month, at start and daily after. -func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) { - for { - if n, err := inv.ForgetOldDurations(ctx); err != nil { - fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err) - } else if n > 0 { - fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor) - } - select { - case <-ctx.Done(): - return - case <-time.After(24 * time.Hour): - } - } -} diff --git a/cmd/mesh-controller/epoch_send_test.go b/cmd/mesh-controller/epoch_send_test.go deleted file mode 100644 index 4fc2e5e4..00000000 --- a/cmd/mesh-controller/epoch_send_test.go +++ /dev/null @@ -1,109 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said -// it reads one, and to no other: an older node-engine refuses a key it does not know, whole. - -// bodiesDelivery records each send as the mesh does, and keeps the bodies. -type bodiesDelivery struct { - recordedDelivery - bodies map[string][]byte -} - -func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { - b.bodies[s.node] = body - return b.recordedDelivery.declare(ctx, s, body) -} - -func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - epoch := uint64(57) - was := epochForActs - epochForActs = func(context.Context) (uint64, error) { return epoch, nil } - t.Cleanup(func() { epochForActs = was }) - - anchor, err := inv.NodeByName(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil { - t.Fatal(err) - } - d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open), d, "", nil); err != nil { - t.Fatal(err) - } - carried := func(node string) (epoch float64, has bool) { - var envelope map[string]any - if err := json.Unmarshal(d.bodies[node], &envelope); err != nil { - t.Fatal(err) - } - epoch, has = envelope["epoch"].(float64) - return epoch, has - } - if e, has := carried("anchor"); !has || e != 57 { - t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"]) - } - if _, has := carried("laptop"); has { - t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"]) - } - - // A new holder of the lease is not a change of the machine: neither reads as behind. - epoch = 58 - would, err := wouldSend(ctx, open, mustNodes(t, open)) - if err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - sent, err := inv.Outstanding(ctx, node) - if err != nil { - t.Fatal(err) - } - if would[node] != sent { - t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node) - } - } -} - -// A process that may not act composes nothing and sends nothing: its number is not taken. -func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - was := epochForActs - epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") } - t.Cleanup(func() { epochForActs = was }) - d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}} - refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open), d, "", nil) - if err != nil { - t.Fatal(err) - } - if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") { - t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused) - } - anchor, _ := open.inventory.NodeByName(ctx, "anchor") - if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 { - t.Fatalf("a controller without the lease took sequence %d", seq) - } - - // And at the send itself: the gate every declaration passes. - gate := link.ActingGate - link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") } - t.Cleanup(func() { link.ActingGate = gate }) - if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil || - !strings.Contains(err.Error(), "lost the lease") { - t.Fatalf("a declaration was let through the gate: %v", err) - } -} diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index 4cb07134..d1985eac 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -8,7 +8,6 @@ import ( "flag" "fmt" "os" - "slices" "sort" "strings" "sync" @@ -18,7 +17,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/compose" snapshot "git.novox.be/novox/mesh-sdk/go/facts" "github.com/novox/mesh-controller/internal/artifacts" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -315,13 +313,6 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot return snapshot.Facts{}, err } m.Assigned = assigned - sent, known, err := inv.SentBuilds(ctx, n.Name) - if err != nil { - return snapshot.Facts{}, err - } - if known && slices.Contains(assigned, broker.RuntimeModule) { - m.NodeTools = sent[broker.RuntimeModule] - } pins, err := inv.PinsFor(ctx, n.Name) if err != nil { return snapshot.Facts{}, err diff --git a/cmd/mesh-controller/failed_wait_provider_test.go b/cmd/mesh-controller/failed_wait_provider_test.go deleted file mode 100644 index 2790e340..00000000 --- a/cmd/mesh-controller/failed_wait_provider_test.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450). -// -// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy -// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the -// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never -// listed on the condition that is open. - -// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check. -var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"} - -// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why. -func judgedRefused() inventory.ResourceHealth { - r := waitingDatabaseFor(refusedWait) - r.State, r.Waits = link.StateUnhealthy, nil - r.Reason = "says it waits for the secret certificate, which db does not declare" - return r -} - -// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list -// it at the provider's unhealthy condition, made urgent by who waits on it. -func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) { - k, _ := withConditionsInMemory(t) - stored := waitingDatabaseFor(refusedWait) - open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()}, - map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}}, - shopFailingBeside(stored)) - provider := conditionOf(open, moduleUnhealthyKey("db", "anchor")) - if len(open) != 1 || provider == nil { - t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+ - "unhealthy alone", openKeysOf(open)) - } - if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") { - t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said) - } - if provider.Severity != conditions.Urgent { - t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity) - } -} - -// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens: -// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when -// open, is said as unhealthy with who waits on it. -func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) { - given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase) - given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)} - by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}) - if !held || by.provider != theDatabase || len(by.waits) > 0 { - t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+ - "held under db on anchor as unhealthy, no waits", held, by.provider, by.waits) - } - if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered { - t.Fatalf("a provider whose wait failed its check is said as waiting for another part") - } -} diff --git a/cmd/mesh-controller/first_builds.go b/cmd/mesh-controller/first_builds.go deleted file mode 100644 index b6c06bfc..00000000 --- a/cmd/mesh-controller/first_builds.go +++ /dev/null @@ -1,277 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "regexp" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// firstBuilds asks the first build of every module a merge adds to a repository no registered module is built from, when -// the merge was announced as made on the operator's Approve of that head (novox/hq ADR 0432, issue 1146): the modules' -// directories are the ones the announcement says hold a module.json at the merge commit, as touchedBy reads them. The -// merge is kept, so the build's outcome is judged by the rule that registers it (mayRegisterFrom, approvedMove). -// -// **Complete in either order, with no timer** (ADR 0426): when mesh-delivery's word is kept first, it is found here; when -// the forge's merge is heard first, the merge is kept as a first merge waiting for the word, and keepingDeliveryMerges -// runs this again when the word is kept (resumeFirstBuilds). A word that says no (a Decline, another head) asks nothing and -// says why. Answers whether the merge was dealt with here. -func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, now time.Time) (bool, error) { - inv := f.open.inventory - repository := m.Owner + "/" + m.Repo - added, why, err := firstBuildsOf(ctx, inv, m, entries, read) - switch { - case errors.Is(err, errFirstNotApproved): - fmt.Printf("%s merged into %s (%.8s) adds %s, and no build is asked: %v\n", repository, m.Base, m.Commit, - strings.Join(addedDirs(m, entries, read), ", "), err) - return true, nil - case errors.Is(err, errNotAnnounced): - // Kept, waiting for mesh-delivery's word: a restart keeps it, and the word's arrival asks the builds. - if kerr := keepFirstMerge(ctx, inv, m, now, true); kerr != nil { - return true, notNow(kerr) - } - fmt.Printf("%s merged into %s (%.8s) adds %s; it is kept, and the first build is asked when mesh-delivery "+ - "announces the merge on the operator's Approve\n", repository, m.Base, m.Commit, - strings.Join(addedDirs(m, entries, read), ", ")) - return true, nil - case err != nil: - return true, notNow(err) - case len(added) == 0: - return false, nil - } - if err := askFirstBuilds(ctx, f.open, m, now, added, why); err != nil { - return true, notNow(err) - } - return true, nil -} - -// keepFirstMerge keeps a first merge as the controller heard it, waiting (no plan answers it). -func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, waitingForWord bool) error { - merged, err := time.Parse(time.RFC3339Nano, m.MergedAt) - if err != nil { - merged = now - } - event, err := json.Marshal(keptMerge{SourceMoved: m, FirstBuild: waitingForWord}) - if err != nil { - return err - } - _, err = inv.AddMerge(ctx, inventory.BatchedMerge{Repository: m.Owner + "/" + m.Repo, Branch: m.Base, Commit: m.Commit, - Merged: merged.UTC(), Heard: now, Event: event}) - return err -} - -// askFirstBuilds keeps the merge as answered and registers its modules from the builder's done keys (novox/hq ADR 0501 -// §3, §6): nothing asks their first builds. The merge kept is the registration's warrant (approvedMove reads it by the -// commit the indexer's trunk value names); a module whose key is not done yet is registered when the key's write wakes -// the take-in (new_modules.go). -func askFirstBuilds(ctx context.Context, open *stores, m link.SourceMoved, now time.Time, added []string, - why string) error { - inv := open.inventory - repository := m.Owner + "/" + m.Repo - if err := keepFirstMerge(ctx, inv, m, now, false); err != nil { - return err - } - fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so %s "+ - "are registered from the builder's done keys, asking no build\n", repository, m.Base, m.Commit, why, - strings.Join(added, ", ")) - // Answered: it no longer waits, so the word's arrival again does nothing twice. - if err := inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false); err != nil { - return err - } - modules, err := newModulesOf(ctx, repository) - switch { - case err != nil: - // Said, not failed: the merge is answered, and the look at every module when a controller starts, or the next - // write of the builder or the indexer for one of them, registers them. - fmt.Printf(" the indexer's modules of %s cannot be read now (%v): they are registered when the builder's next "+ - "write of one wakes the take-in\n", repository, err) - case len(modules) > 0: - inBackground(func() { takeNewModules(context.WithoutCancel(ctx), open, modules...) }) - } - return nil -} - -// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull -// request's head: the other order of the same two events. -func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error { - actingOnMerges.Lock() - defer actingOnMerges.Unlock() - inv := f.open.inventory - waiting, err := inv.WaitingMerges(ctx) - if err != nil { - return notNow(err) - } - for _, w := range waiting { - k := announcedOf(w) - if !k.FirstBuild || len(k.Held) > 0 || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number || - k.HeadSHA != d.HeadSHA { - continue - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return notNow(err) - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return notNow(err) - } - added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read) - switch { - case errors.Is(err, errFirstNotApproved): - fmt.Printf("%s/%s merged into %s (%.8s) was waiting for mesh-delivery's word, and no build is asked: %v\n", - k.Owner, k.Repo, k.Base, k.Commit, err) - if aerr := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); aerr != nil { - return notNow(aerr) - } - case err != nil: - return notNow(err) - case len(added) > 0: - if err := askFirstBuilds(ctx, f.open, k.SourceMoved, time.Now().UTC(), added, why); err != nil { - return notNow(err) - } - } - } - return nil -} - -// errFirstNotApproved says a merge into a repository the mesh builds nothing from is not one the operator approved. -var errFirstNotApproved = errors.New("it is no merge made on the operator's Approve of its head, so a hand `build` " + - "registers a module from a repository the mesh does not build yet") - -// addedDirs are the module directories the merge adds to a repository no registered module is built from, none where -// the merge is not into the trunk or some module is built from the repository already. -func addedDirs(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []string { - if m.Base != "main" { - return nil - } - for _, e := range entries { - if !e.Provided && sameRepository(e.Source.Repository, m) { - return nil - } - } - _, added, _ := touchedBy(nil, entries, m, read) - return added -} - -// firstBuildsOf is the directories whose first build a merge asks, and the warrant it was made on; none when the merge -// is no first merge into a repository the mesh builds nothing from. errFirstNotApproved when it is and the word on record -// is no Approve of its head; errNotAnnounced when mesh-delivery's word is not kept yet. -func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) ([]string, string, error) { - added := addedDirs(m, entries, read) - if len(added) == 0 { - return nil, "", nil - } - why, err := approvedAnnounced(ctx, inv, m.Owner, m.Repo, keptMerge{SourceMoved: m}) - switch { - case err == nil: - return added, why, nil - case errors.Is(err, errWordNotAnApprove): - return nil, "", fmt.Errorf("%w (%v)", errFirstNotApproved, err) - } - // Not announced yet, or the bus or the store could not be asked: the caller waits or tries again, and asks nothing. - return nil, "", err -} - -// heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read, -// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173). Read from a merge kept -// before R4e, never written since: nothing asks a first build (ADR 0501 §6), and a controller that starts answers such a -// merge and registers its modules from their keys (answerHeldMerges). -type heldFirstBuild struct { - Module string `json:"module"` - Path string `json:"path"` - Word string `json:"word"` - At time.Time `json:"at"` -} - -// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which -// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words). -var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`) - -// refusedWord is the word a build's failure says the reader did not know; empty when it says none. -func refusedWord(failed string) string { - if m := unknownWord.FindStringSubmatch(failed); m != nil { - return m[1] - } - return "" -} - -// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word. -const probeFirstBuildsID = "D18" - -// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises. -const kindFirstBuildUnasked = "first-build-unasked" - -// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before -// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds -// at once, and the merge is said, never dropped, once it is past this (issue 1146). -const firstBuildWaitBound = 10 * time.Minute - -// firstBuildObservations are the conditions of the first merges waiting past the bound, one for each; a merge answered is -// no longer listed as waiting, so its condition clears with the next look. -func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []conditions.Observation { - var out []conditions.Observation - for _, w := range waiting { - k := announcedOf(w) - if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound { - continue - } - repository := k.Owner + "/" + k.Repo - for_ := now.Sub(w.Heard).Round(time.Minute) - out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, - ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), Kind: kindFirstBuildUnasked, Severity: conditions.Warning, - Summary: fmt.Sprintf("the merge %.8s of %s adds modules the mesh builds nothing from yet, and has waited %s for "+ - "mesh-delivery's announcement of the operator's Approve of its head: no build is asked", k.Commit, repository, for_), - Said: fmt.Sprintf("%s at %s waits for mesh-delivery's word on pull request %d (head %.8s)", repository, k.Commit, - k.Number, k.HeadSHA), - Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository), - Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+ - "only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+ - "announcement has come in %s. The merge is kept, and the modules are registered from the builder's keys the moment "+ - "the announcement is kept.", - k.Commit, repository, for_), - Resolved: "mesh-delivery announces the merge, or the operator registers the modules at the controller's terminal", - }) - } - return out -} - -// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid. -func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - waiting, err := d.open.inventory.WaitingMerges(ctx) - if err != nil { - return nil, err - } - now := time.Now().UTC() - return append(firstBuildObservations(waiting, now), newModuleObservations(now)...), nil -} - -// answerHeldMerges answers every merge a controller of before held for a build-agent that reads its new modules' -// manifests (issue 1173): nothing asks their first builds any more, and the look at every module when this controller -// starts registers them from their keys (ADR 0501 §6, §7). -func answerHeldMerges(ctx context.Context, inv *inventory.Inventory) { - waiting, err := inv.WaitingMerges(ctx) - if err != nil { - fmt.Printf("the merges held for a newer build-agent could not be read: %v\n", err) - return - } - for _, w := range waiting { - if k := announcedOf(w); len(k.Held) > 0 { - if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil { - fmt.Printf("the merge %.8s of %s held for a newer build-agent could not be answered: %v\n", w.Commit, - w.Repository, err) - continue - } - fmt.Printf("the merge %.8s of %s was held for a newer build-agent; its modules are registered from the "+ - "builder's keys instead\n", w.Commit, w.Repository) - } - } -} diff --git a/cmd/mesh-controller/first_builds_integration_test.go b/cmd/mesh-controller/first_builds_integration_test.go deleted file mode 100644 index 4608f763..00000000 --- a/cmd/mesh-controller/first_builds_integration_test.go +++ /dev/null @@ -1,152 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// 1146/AC7 on keys (ADR 0501 §3, §6; issue 1851, A5), whole against a real bus and the store, in both orders of the two -// events: mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's -// merge, heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the -// controller consumes with the router's record of the ask in its bucket, come in either order; each way the module the -// merge commit holds is registered from the builder's done key of it, and nothing asks a build. Without the Approve in -// the record it is not registered. Store-backed: skips without one. -func TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey(t *testing.T) { - const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000" - for _, c := range []struct { - what string - wordFirst bool - approved bool // the router's record holds the operator's Approve of the merged head - wantRegister bool - }{ - {what: "the forge's merge before mesh-delivery's word", approved: true, wantRegister: true}, - {what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegister: true}, - {what: "a word whose record holds no Approve", wordFirst: true}, - } { - t.Run(c.what, func(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - noBuildAsked(t) - b := aBuilderOnTheBus(t) - js := b.js - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1", - DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")}) - if err != nil { - t.Fatal(err) - } - a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head}) - if c.approved { - record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant}) - if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil { - t.Fatal(err) - } - } - // The indexer's value and the builder's done key of the module the merge commit holds. - fp := fpOf('9') - manifest := b.indexed(t, "mesh-delivery-view", "1", "novox/mesh-delivery", ".", commit, fp) - b.built(t, "mesh-delivery-view", "1", fp, manifest, sdkdelivery.BuildDone) - - following := following{open: open} - server := link.ConnectNats(js, nil, noListener{}) - if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims, kept: following.resumeFirstBuilds}); err != nil { - t.Fatal(err) - } - if err := broker.AssertServingConsumers(js); err != nil { - t.Fatal(err) - } - serving, stop := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { defer close(done); _ = server.Serve(serving) }() - t.Cleanup(func() { stop(); <-done }) - - word := func() { - t.Helper() - body, _ := json.Marshal(a.merge) - if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil { - t.Fatal(err) - } - becomes(t, 15*time.Second, "the word being kept", func() bool { - _, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, head) - return err == nil && kept - }) - time.Sleep(500 * time.Millisecond) // the keeper's hook runs as the word is kept - } - forge := func() { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7, - MergedAt: time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"}, - ModuleDirs: []string{"."}, ModuleDirsSaid: true} - if err := following.SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - } - if c.wordFirst { - word() - forge() - } else { - forge() - time.Sleep(300 * time.Millisecond) - if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known { - t.Fatal("1146/AC7: registered before mesh-delivery's word was kept") - } - word() - } - if !c.wantRegister { - time.Sleep(time.Second) - if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known { - t.Fatal("1146/AC7: a merge whose record holds no Approve registered its module") - } - return - } - becomes(t, 10*time.Second, "the module registered", func() bool { - _, known := inventoryHolds(t, inv, "mesh-delivery-view") - return known - }) - src, err := inv.SourceOf(ctx, "mesh-delivery-view") - if err != nil || src.Repository != "novox/mesh-delivery" { - t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err) - } - }) - } -} - -func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) { - t.Helper() - entries, err := inv.Catalogued(t.Context()) - if err != nil { - t.Fatal(err) - } - for _, e := range entries { - if e.Manifest.Module == module { - return e, true - } - } - return inventory.Entry{}, false -} diff --git a/cmd/mesh-controller/first_builds_test.go b/cmd/mesh-controller/first_builds_test.go deleted file mode 100644 index b0c4d009..00000000 --- a/cmd/mesh-controller/first_builds_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// 1146/AC6: the first merge into a repository no registered module is built from asks the first build of every module -// the merge commit holds when it was made on the operator's Approve of that head, and asks nothing without it. -func TestAFirstMergeIntoANewRepositoryAsksItsModulesBuildsOnlyOnTheApprove(t *testing.T) { - const head = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" - now := time.Date(2026, 10, 19, 0, 6, 0, 0, time.UTC) - mergedAt := now.Add(-time.Hour).Format(time.RFC3339Nano) - heard := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: "0177ea7a", - Number: 1, MergedAt: mergedAt, ModuleDirsSaid: true, ModuleDirs: []string{"watchman", "view"}, - Paths: []string{"view/module.json", "watchman/module.json", "README.md"}} - built := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git"}} - for _, c := range []struct { - what string - change func(*anApprovedMerge, *link.SourceMoved, *[]inventory.Entry) - unkept bool - dirs []string - refused bool - waits bool - }{ - {what: "the Approve of the merged head", dirs: []string{"view", "watchman"}}, - {what: "a Decline", refused: true, change: func(a *anApprovedMerge, _ *link.SourceMoved, _ *[]inventory.Entry) { - a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline" - }}, - {what: "no word of mesh-delivery yet, however long ago the merge was made", unkept: true, waits: true}, - {what: "a module already built from the repository", change: func(_ *anApprovedMerge, _ *link.SourceMoved, e *[]inventory.Entry) { - *e = append(*e, inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}}) - }}, - {what: "a merge into another branch", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { m.Base = "feat" }}, - {what: "a merge that holds no module.json", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { - m.ModuleDirs, m.Paths = nil, []string{"README.md"} - }}, - } { - t.Run(c.what, func(t *testing.T) { - a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head}) - a.merge.Number = 1 - m, entries := heard, []inventory.Entry{built} - if c.change != nil { - c.change(&a, &m, &entries) - } - wasKept, wasAsk := keptDeliveryMerge, routerAskOf - t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) - keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) { - return a.merge, !c.unkept, nil - } - routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) { - return &a.ask, &a.warrant, nil - } - dirs, _, err := firstBuildsOf(t.Context(), nil, m, entries, nil) - switch { - case c.waits && !errors.Is(err, errNotAnnounced): - t.Fatalf("1146/AC6: %s: want a wait for the word, got %v", c.what, err) - case c.refused && !errors.Is(err, errFirstNotApproved): - t.Fatalf("1146/AC6: %s: want the Approve missing, got %v (dirs %v)", c.what, err, dirs) - case !c.refused && !c.waits && err != nil: - t.Fatalf("1146/AC6: %s: %v", c.what, err) - case !slices.Equal(dirs, c.dirs): - t.Fatalf("1146/AC6: %s: first builds asked of %v, want %v", c.what, dirs, c.dirs) - } - }) - } -} - -// 1146/AC8: a first merge waiting past the bound for mesh-delivery's word is said, naming the repository, the commit and -// the missing word; one inside the bound, one that is no first merge, and one answered (no longer waiting) are not. -func TestAFirstMergeWaitingPastItsBoundIsSaidAndClearsWhenAnswered(t *testing.T) { - now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC) - waiting := func(commit string, first bool, heard time.Time) inventory.BatchedMerge { - event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", - Commit: commit, Number: 1, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: first}) - return inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, Heard: heard, Event: event} - } - got := firstBuildObservations([]inventory.BatchedMerge{ - waiting("0177ea7a00", true, now.Add(-firstBuildWaitBound-time.Minute)), - waiting("1111111100", true, now.Add(-time.Minute)), - waiting("2222222200", false, now.Add(-time.Hour)), - }, now) - if len(got) != 1 { - t.Fatalf("1146/AC8: %d conditions raised, want the one merge past the bound: %+v", len(got), got) - } - o := got[0] - if o.Kind != kindFirstBuildUnasked || o.Scope != conditions.ScopeMerge || o.Severity != conditions.Warning || - !strings.Contains(o.Summary, "novox/mesh-delivery") || !strings.Contains(o.Summary, "0177ea7a") || - !strings.Contains(o.Summary, "announcement of the operator's Approve") { - t.Fatalf("1146/AC8: the condition is %+v", o) - } - // Answered, the merge is no longer listed as waiting: nothing is observed, and the condition clears. - if cleared := firstBuildObservations(nil, now); len(cleared) != 0 { - t.Fatalf("1146/AC8: an answered merge still raises %+v", cleared) - } -} diff --git a/cmd/mesh-controller/fixclass_test.go b/cmd/mesh-controller/fixclass_test.go deleted file mode 100644 index 9541c005..00000000 --- a/cmd/mesh-controller/fixclass_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the -// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core, -// leaf or none, so mesh-delivery can pass it on with the fix's delivery. -func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) { - const repo = "http://forge.internal:20000/novox/mesh-catalog.git" - entry := func(name, path string) inventory.Entry { - e := fromRepo(name, repo, path) - e.Source.BuiltFrom = "old" - e.On = []string{"anchor"} - return e - } - gitea := entry("gitea", "modules/gitea") - resolver := entry("dnsmasq", "modules/dnsmasq") - resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}} - delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery") - delivery.Manifest.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} - webApp := entry("photos", "modules/photos") - entries := []inventory.Entry{gitea, resolver, delivery, webApp} - // photos stands on gitea: a change to gitea moves photos as its dependent. - edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}} - policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true } - plan := func(paths ...string) link.ChangePlan { - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths} - return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy) - } - - for _, c := range []struct { - why string - paths []string - want string - }{ - {"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf}, - {"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore}, - {"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore}, - {"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore}, - {"nothing any module's build reads", []string{"README.md"}, "none"}, - {"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf}, - {"a new module on the controller's own path, by its own class", []string{"next/mesh-host/module.json"}, inventory.ClassCore}, - {"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore}, - } { - if got := plan(c.paths...).Class; got != c.want { - t.Errorf("%s: class %q, want %q", c.why, got, c.want) - } - } -} diff --git a/cmd/mesh-controller/delivery_changes.go b/cmd/mesh-controller/forge_read.go similarity index 54% rename from cmd/mesh-controller/delivery_changes.go rename to cmd/mesh-controller/forge_read.go index 0e2a2966..f4b85673 100644 --- a/cmd/mesh-controller/delivery_changes.go +++ b/cmd/mesh-controller/forge_read.go @@ -17,9 +17,8 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// What a change says of the manifests it touches, read from the forge (novox/hq ADR 0419 clause 3, the lead's settlement -// of issue 977 question 1): the controller asks the forge for each changed module.json at the commit and at what it -// replaced, once per commit. The composed check's result is not used: the plan is cut after the merge. +// What the forge says of a change's manifests (novox/hq ADR 0502): read through the forge's module on the bus, once per +// commit. Kept by release 6 for the pull request's declared config; the delivery plan's reads of it are gone. // changeSource is one change to a repository whose manifests are read: the commit after it and, when known, what it // replaced (a pull request's base commit or branch). An empty Old is the first parent of New: the merge's base. @@ -63,24 +62,6 @@ var manifestAt = func(ctx context.Context, owner, repo, file, ref string) (catal return m, true, nil } -// parentOf is the first parent of a commit, from the forge. A variable so a test needs no forge. -var parentOf = func(ctx context.Context, owner, repo, commit string) (string, error) { - var found struct { - Result struct { - Parents []struct { - SHA string `json:"sha"` - } `json:"parents"` - } `json:"result"` - } - if _, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(commit), &found); err != nil { - return "", err - } - if len(found.Result.Parents) == 0 { - return "", nil - } - return found.Result.Parents[0].SHA, nil -} - func escapedPath(p string) string { parts := strings.Split(p, "/") for i := range parts { @@ -114,12 +95,6 @@ func askForgeApi(ctx context.Context, apiPath string, into any) (bool, error) { return false, nil } -// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit. -var ( - manifestsRead = map[string]manifestRead{} - manifestsReadMu sync.Mutex -) - type manifestRead struct { manifest catalogue.Manifest exists bool @@ -147,96 +122,11 @@ func manifestCached(ctx context.Context, owner, repo, file, ref string) (catalog return m, exists, err } -// verbsServed are the seat verbs a manifest serves, as ".": those of the seats it defines, those of the -// seats it claims, and its own tools as ".". -func verbsServed(m catalogue.Manifest) []string { - var out []string - for _, s := range m.DefinesSeats { - for _, v := range s.Serves { - out = appendNew(out, s.Name+"."+v.Name) - } - } - for _, c := range m.Claims { - for _, v := range c.ServesFor(m) { - out = appendNew(out, c.Name+"."+v) - } - } - for _, t := range m.Tools { - out = appendNew(out, m.Module+"."+t) - } - return out -} - -// verbsInvoked are the verbs a manifest calls, as "." or ".". -func verbsInvoked(m catalogue.Manifest) []string { - var out []string - for _, i := range m.Invokes { - if i = strings.TrimPrefix(i, "seat:"); i != "*" && i != "" { - out = appendNew(out, i) - } - } - return out -} - -func added(now, was []string) []string { - var out []string - for _, v := range now { - if !slices.Contains(was, v) { - out = append(out, v) - } - } - return out -} - -// changesOf reads, for each manifest the changes touch, what they add to the verbs its module serves and invokes, by -// module. A change whose paths are unsaid or truncated names no manifest: nothing is read for it. An error is the -// forge's, never read as "no change": the plan is not cut on a guess. -func changesOf(ctx context.Context, sources []changeSource) (map[string]moduleChange, error) { - out := map[string]moduleChange{} - for _, s := range sources { - if s.Truncated { - continue - } - for _, p := range s.Paths { - if path.Base(p) != "module.json" || slices.Contains(s.Removed, p) { - continue - } - now, exists, err := manifestCached(ctx, s.Owner, s.Repo, p, s.New) - if err != nil { - return nil, err - } - if !exists || now.Module == "" { - continue - } - old := s.Old - if old == "" { - first := s.Since - if first == "" { - first = s.New - } - if old, err = parentOf(ctx, s.Owner, s.Repo, first); err != nil { - return nil, err - } - } - var was catalogue.Manifest - if old != "" { - if was, _, err = manifestCached(ctx, s.Owner, s.Repo, p, old); err != nil { - return nil, err - } - } - c := out[now.Module] - c.Manifest = true - for _, v := range added(verbsServed(now), verbsServed(was)) { - c.Serves = appendNew(c.Serves, v) - } - for _, v := range added(verbsInvoked(now), verbsInvoked(was)) { - c.Invokes = appendNew(c.Invokes, v) - } - out[now.Module] = c - } - } - return out, nil -} +// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit. +var ( + manifestsRead = map[string]manifestRead{} + manifestsReadMu sync.Mutex +) // truncatedPaths are the manifests a change that names only a beginning of its files is read as changing: those of the // modules it moves that are built from its repository. Said, never skipped: no edge may be lost to a long list. @@ -251,21 +141,3 @@ func truncatedPaths(entries []inventory.Entry, owner, repo string, moved []strin owner, repo, strings.Join(out, ", ")) return out } - -// changeSourcesOf are a group's members as changes to read: each head over what its pull request merges into. -func changeSourcesOf(members []orderMember, entries []inventory.Entry, reach map[string]orderReach) []changeSource { - var out []changeSource - for _, m := range members { - p := m.pull() - if p.PathsTruncated { - p.Paths, p.PathsTruncated = truncatedPaths(entries, p.Owner, p.Repo, reach[m.ID].Moved), false - } - old := p.BaseSHA - if old == "" { - old = p.Base - } - out = append(out, changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old, Paths: p.Paths, - Truncated: p.PathsTruncated, Removed: p.Removed}) - } - return out -} diff --git a/cmd/mesh-controller/found_wait_integration_test.go b/cmd/mesh-controller/found_wait_integration_test.go deleted file mode 100644 index 1a1f193c..00000000 --- a/cmd/mesh-controller/found_wait_integration_test.go +++ /dev/null @@ -1,127 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - for _, c := range []struct { - name string - found time.Duration // when the directory was found, against now - passes bool - }{ - {"found a day before the send", -24 * time.Hour, true}, - {"found by this send", time.Hour, false}, - } { - t.Run(c.name, func(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - // The used-as-found condition, raised after the send (its second statement): its own kind, never a - // fault the gate reads as the build's. - f.judged, f.openErr = true, nil - f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound, - Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, - Raised: time.Now()}) - f.health = map[string]inventory.NodeHealth{} - for _, n := range []string{"anchor", "laptop"} { - f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - foundDirectory("app", time.Now().Add(c.found)), - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}} - } - return f, err - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - v, found, err := inv.GateOf(ctx, "build-app-c2") - if c.passes { - if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v) - } - if !strings.Contains(v.Why+p.Note, "hand it over") { - t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note) - } - return - } - if p.State == inventory.PlanDone { - t.Fatalf("a directory this send found let the walk through: %s", p.Note) - } - }) - } -} - -func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} - for i := 0; i < 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil { - t.Fatal(err) - } - } - open, _ := k.Open(ctx) - var got *conditions.Condition - for i, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - got = &open[i] - } - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault: %+v", c) - } - } - if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent || - !strings.Contains(got.Summary, "notes.data") { - t.Fatalf("the condition: %+v", got) - } - // Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait. - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent { - t.Fatal("a directory used as found became urgent") - } - } - if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - t.Fatal("not cleared once handed over") - } - } -} diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go deleted file mode 100644 index 54621738..00000000 --- a/cmd/mesh-controller/found_wait_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the -// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an -// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds. -func foundDirectory(module string, since time.Time) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory, - Target: "/srv/" + module, State: link.StateUnhealthy, Since: since, - Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " + - "not given it — `nox node hand-over laptop ` on the control-node, with its path, hands it to the mesh"} -} - -func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - now := time.Now() - sent := now.Add(-time.Minute) - f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, - Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}} - h, why := moduleHealthWord("notes", "laptop", sent, f) - if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") { - t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why) - } - // Found by this very send: the send brought it, and it is not passed. - f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, - Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}} - if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet { - t.Fatalf("a directory this send found reads %v %q; want not yet", h, why) - } - // A container down beside the old wait is a fault, as before. - f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{ - foundDirectory("notes", sent.Add(-time.Hour)), - {Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}} - if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet { - t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why) - } -} - -// The whole walk: a module whose directory was used as found long before still gets its fix to every machine, -// its pass kept and the wait said; a directory this very send found holds it and puts it back. - -// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over. diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go deleted file mode 100644 index 30a7499d..00000000 --- a/cmd/mesh-controller/gate.go +++ /dev/null @@ -1,1651 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sort" - "strings" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/micro" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0236, to-be 45 -// §8, ADR 0227 rule 8). -// -// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once -// that machine reported it applied. A build that applies and then does nothing, crashes, serves no -// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by -// the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third -// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the -// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states -// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 -// decision 3). -// Only then are the rest sent. -// -// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed -// at its gate — registration refuses it and nothing sends it again on its own — the module's registered -// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary -// path, again. Once per build: the verdict is written before the send, and a verdict once written is -// not written over. Said as a condition, urgent for the core and when it could not be put back, and as -// the event `rolled-back`. - -// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. -var ( - // gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the - // gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR - // 0352). Readings gateEvery apart, the first at the apply, put three inside it. - gateWithin = 2 * time.Minute - // gateBound is how long after the apply the build may take to become healthy. - gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at - // the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the - // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 - // decision 4 as ADR 0304 decision 3 changed it, issue 524). - gatePasses = 3 - gateLeafPasses = 1 - gateLeafSettles = false - gateEvery = 40 * time.Second - // gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its - // start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either - // counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR - // 0351 decisions 1 and 4). - gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin) -) - -// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's -// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's -// delivery budget; how many readings its gate takes is gateRuleOf's. -func coreModule(module string, m catalogue.Manifest) bool { - if onTheOwnPath(module, m) { - return true - } - for _, c := range m.Claims { - if c.Name == resolverSeat { - return true - } - } - return coreComponent(module) != "" -} - -// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the -// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it; -// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading -// (novox/hq ADR 0352). said names the rule in the gate's text. -type gateRule struct { - passes int - within time.Duration - witnessed bool - said string -} - -// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its -// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send, -// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never -// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner, -// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4); -// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304 -// decision 3). -func gateRuleOf(module string, m catalogue.Manifest) gateRule { - switch { - case coreComponent(module) == lease.ComponentController: - return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " + - "after the node-engine's witness window (novox/hq ADR 0351)"} - case coreComponent(module) == lease.ComponentNodeTools: - return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " + - "after the node-engine's witness window (novox/hq ADR 0351)"} - case m.ClaimsSeat(catalogue.DeliverySeat): - return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} - case coreModule(module, m): - return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+ - "or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)} - } - r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} - if gateLeafSettles { - r.within = gateWithin - r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send", - gateLeafPasses, gateWithin) - } - return r -} - -// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the -// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A -// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it -// applied (applied, as the gate first read it) and, for the controller, before the new controller took the -// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. -// A controller that starts again takes the lease again, and so moves its floor on. Pure. -func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, - holder *lease.Holder) time.Time { - floor := since - if !rule.witnessed { - return floor - } - for _, n := range machines { - start := since - if at, read := applied[n]; read && at.After(start) { - start = at - } - if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) { - start = holder.Taken - } - if end := start.Add(gateWitnessWindow); end.After(floor) { - floor = end - } - } - return floor -} - -// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions -// are raised by a plan as it judges, and kept or cleared by the probe on every run. -const gateProbe = "DG" - -// The kinds a gate raises. -const ( - kindRolledBack = "rolled-back" - kindRollbackFailed = "rollback-failed" -) - -// coreComponent is the core component a module is, as a witness names it — controller, node-engine, -// node-tools — or empty: the core is judged by its health definitions, anything else by its own health. -func coreComponent(module string) string { - switch module { - case catalogue.ControllerSeatName: - return lease.ComponentController - case hostModule: - return lease.ComponentEngine - case broker.RuntimeModule: - return lease.ComponentNodeTools - } - return "" -} - -// health is a judging's word on one machine. -type health int - -const ( - healthGood health = iota - // healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for - // one person's new login (personWait). The build did what it should; no bound a machine can meet ends - // the wait. It counts as a pass, and the gate carries the reading along in its verdict. - healthPerson - // healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy - // provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a - // build for something it did not do. - healthWaiting - healthNotYet - healthBroken - // healthSuperseded is a judging that cannot go on: the machine was sent another build of the module - // after the gate's send (novox/hq issue 352). No verdict on the build judged, and nothing put back. - healthSuperseded -) - -// served is what one machine's node tools answered the bus's discovery with. -type served struct { - runtime bool - tools map[string]bool -} - -// gateFacts is what one judging reads, gathered once for every machine it judges. -type gateFacts struct { - now time.Time - reports map[string]inventory.Reported - // engines is each machine's node-engine build as it last reported it. - engines map[string]string - // served is what the bus's discovery answered; servedErr why it could not be asked. - served map[string]served - servedErr error - // open are the open conditions; openErr why they could not be read (nil keeper: not judged). - open []conditions.Condition - openErr error - judged bool - // rolledBack is what each machine's witnesses say they decided. - rolledBack map[string][]lease.Rollback - // holder is who holds the controller lease, for judging the controller. - holder *lease.Holder - holderErr error - // health is each machine's newest health statement (ADR 0240); a machine absent never stated one. - // healthErr is why they could not be read. - health map[string]inventory.NodeHealth - healthErr error - // heldOn is, per "@", the provider its findings are held under (ADR 0240 rule 5). - heldOn map[string]heldReading - // groupsAdded is, per module, whether the move judged puts an account in a group its previous build did - // not (issue 318 review): the only move whose wait for a new login is excused. - groupsAdded map[string]bool - // waits is what the controller holds to check a module's wait for the operator (novox/hq ADR 0283): the - // manifest of each module's build judged, and the secrets given on each machine. - waits operatorWaitFacts - // sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a - // report is held against it, never against the send made last. sentBuilds is what each machine was - // last sent of every module, and judged the commit of each module this gate judges: a machine last - // sent another build of the module is not running the build judged. - sent map[string]inventory.SentDeclaration - sentBuilds map[string]map[string]string - commits map[string]string - // provisioned answers each machine's provisioner client's newest report of a module (novox/hq ADR 0349 §15): - // a module the gate's send handed to the provisioner is healthy only once it reported that build placed. Nil - // reads as nothing said. - provisioned inventory.ProvLookup -} - -// heldReading is the provider a module's findings are held under, as " on ", and, when that -// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue -// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283 -// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here. -type heldReading struct { - on, waits string -} - -// heldReadings is, per "@" in every machine's newest statement, what its findings are held under. -func heldReadings(hold *holding) map[string]heldReading { - out := map[string]heldReading{} - for machine := range hold.healths { - for module, by := range hold.heldModules(machine) { - reading := heldReading{on: by.provider.Module + " on " + by.provider.Node} - if len(by.waits) > 0 { - reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits) - } - out[module+"@"+machine] = reading - } - } - return out -} - -// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that -// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the -// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer -// send another plan had just made there, and failed three builds the machine had reported healthy as -// "has not reported on what it was sent". -func (f gateFacts) reportedOn(machine string, r inventory.Reported) bool { - if sent, kept := f.sent[machine]; kept { - return sent.ReportsOn(r) - } - return r.Current -} - -// supersededOn says the machine was last sent another build of the module than the one this gate judges -// (novox/hq issue 352): the judging cannot go on, whatever the machine reports. On 2026-10-09 a controller -// put back by one gate judged another gate's newer controller build passed on the same machine, reading -// the put-back build's health as the newer one's. -func (f gateFacts) supersededOn(module, machine string) (string, bool) { - judged, known := f.commits[module] - sent, has := f.sentBuilds[machine][module] - if !known || !has || judged == "" || sent == "" || sameCommit(sent, judged) { - return "", false - } - return fmt.Sprintf("%s was sent %s %s after this gate's %s: the build judged no longer runs there, and "+ - "this judging is superseded by that send's", machine, module, short(sent), short(judged)), true -} - -// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A -// variable so a test can hand a judging its facts. -var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - inv := open.inventory - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: witnessed.all(), provisioned: provLookup(ctx)} - reports, err := inv.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - } - nodes, err := inv.Nodes(ctx) - if err != nil { - return f, err - } - for _, n := range nodes { - f.engines[n.Name] = n.HostVersion - } - // What each machine says of its long-running resources (ADR 0240): unreadable is said, never read as - // healthy. - f.health, f.healthErr = inv.Healths(ctx) - if d := doctorFrom; d != nil { - if d.keeper != nil { - f.judged = true - f.open, f.openErr = d.keeper.Open(ctx) - } - if d.js != nil { - f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn()) - } else { - f.servedErr = errors.New("this controller has no bus to ask") - } - } else { - f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what") - } - // Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail. - if f.healthErr == nil && f.openErr == nil { - if hold, err := readHolding(ctx, inv, f.open); err == nil { - f.heldOn = heldReadings(hold) - } - } - if theLease != nil { - h, found, err := theLease.holder(ctx) - switch { - case err != nil: - f.holderErr = err - case found: - f.holder = &h - } - if component != lease.ComponentController && f.holderErr != nil { - f.holderErr = nil // read only for the controller's own judging - } - } - return f, nil -} - -// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy, -// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed -// what it was sent. Pure. -func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) { - // A witness's verdict made since the build was sent: the build failed its health there. One that - // could not judge at all (unwitnessed) is not a verdict on the build. - for _, r := range f.rolledBack[machine] { - if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) { - continue - } - return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component, - short(r.From), r.Outcome, r.Why) - } - if why, superseded := f.supersededOn(module, machine); superseded { - return healthSuperseded, why - } - r, said := f.reports[machine] - switch { - case !said || r.At == nil || !f.reportedOn(machine, r): - return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine) - case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused: - return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome) - case r.Outcome != inventory.OutcomeApplied: - return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome) - } - // **And its files placed** (novox/hq ADR 0349 §15): a module the send handed to the provisioner is judged only - // once that machine's client reported the build's templates placed, so a module whose files are not yet written - // is never judged; a failure there is the build's on that machine. - if h, why := judgeProvisioned(module, machine, r, f); h != healthGood { - return h, why - } - // **No new condition about it**: about the machine itself, or naming the module on that machine, - // raised since the judging began. The gate's own are not evidence about the build. A fault that was - // there at the send and reopened since is not new; one that had cleared before the send and came back - // after it is (OpenAt, novox/hq issue 348). - if f.judged { - if f.openErr != nil { - return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " + - firstLine(f.openErr.Error()) - } - for _, c := range f.open { - // A wait for a person's new login, for a directory used as found to be handed over, or for the - // operator's secret or setting, is the module's reading, not a fault raised since the send: the gate - // reads it from the statement below (ADR 0254, novox/hq issue 339, ADR 0283). - if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound || - c.Kind == kindNeedsOperator { - continue - } - onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || - (c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine) - if !onIt { - continue - } - // A module's own health condition names it whole, its name's dots and all (ADR 0240). - ownHealth := c.Subject.Scope == conditions.ScopeModule && c.Subject.ID == module+"."+machine - if c.Subject.Scope == conditions.ScopeMachine || ownHealth || slices.Contains(strings.Split(c.Subject.ID, "."), module) { - return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary) - } - } - } - switch component { - case lease.ComponentEngine: - // The node-engine has reported its current declaration under its own build. - want := deliveredVersions(m) - if len(want) > 0 && !slices.Contains(want, f.engines[machine]) { - return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine, - orNotKnown(f.engines[machine]), strings.Join(want, " or ")) - } - case lease.ComponentNodeTools: - // The node tools are announced and answer. - if f.servedErr != nil { - return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error()) - } - if !f.served[machine].runtime { - return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine) - } - case lease.ComponentController: - // The new controller holds the lease and says it is ready. - switch { - case f.holderErr != nil: - return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error()) - case f.holder == nil: - return healthNotYet, "no controller holds the lease" - case f.holder.Taken.Before(since.Add(-time.Minute)): - return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build", - f.holder.Taken.UTC().Format(time.RFC3339)) - case f.holder.Health == nil || !f.holder.Health.Ready: - why := "the controller holding the lease does not say it is ready" - if f.holder.Health != nil && f.holder.Health.Why != "" { - why += ": " + f.holder.Health.Why - } - return healthNotYet, why - } - default: - // A module's tools answer, where it has any and the machine runs the node tools that serve them. - if len(m.Tools) > 0 { - if f.servedErr != nil { - return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error()) - } - if s := f.served[machine]; s.runtime && !s.tools[module] { - return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module) - } - } - // **And what it runs is stated healthy** (ADR 0240 §4): every long-running resource of it on that - // machine, in a statement heard since the send. A resource still starting makes the judging wait. - if h, why := moduleHealthWord(module, machine, since, f); h != healthGood { - return h, why - } - } - return healthGood, "" -} - -// judgeProvisioned is whether a module's files are placed on a machine, for a module the gate's send handed to the -// provisioner there: healthy for any other. -func judgeProvisioned(module, machine string, r inventory.Reported, f gateFacts) (health, string) { - sent, kept := f.sent[machine] - digest := sent.Provisioned[module] - if !kept || digest == "" { - return healthGood, "" - } - at := time.Time{} - if r.At != nil { - at = *r.At - } - one := inventory.SentDeclaration{Digest: sent.Digest, At: sent.At, Provisioned: map[string]string{module: digest}} - got, ok := inventory.ProvisionedSend(machine, one, inventory.AppliedReport{At: at, - Outcome: inventory.OutcomeApplied}, true, f.provisioned) - switch { - case !ok: - return healthNotYet, fmt.Sprintf("the provisioner on %s has not reported %s's files placed for the new "+ - "build", machine, module) - case got.Outcome != inventory.OutcomeApplied: - return healthBroken, fmt.Sprintf("the provisioner on %s did not place %s's files: %s", machine, module, - got.Outcome) - } - return healthGood, "" -} - -// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is -// the judging's facts without those conditions, on what each names among the modules the send moved, -// and whole what holds the machine back as a whole. -type machineWord struct { - facts gateFacts - on map[string]string - whole string - // waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits. - waiting string -} - -// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not -// yet reported applying the node tools it was last sent. -const kindCoreBehind = "core-behind" - -// aboutTheMachine sorts the conditions raised about a machine itself since a send was made there -// (novox/hq issue 281). The gate read every one of them as the module's it was kept on: a machine-level -// condition caused by anything else in the send — or by a tier sent one module at a time — failed that -// module, at the bound, with a reason that was never about it. -// -// - one naming a module the send moved is that module's; -// - the core being behind (D10) is the core's: of the node-engine or the node tools when the send -// moved them — inside their settle window, which is the gate's bound — and otherwise no evidence about -// what was sent: a send not yet applied the machine's reports already say, and a newer core build -// that no plan sends is not this send's; -// - anything else holds the machine back as a whole: everything the send moved there waits on it, and -// fails with it, together, at the bound, with that reason. -// -// Pure. -func aboutTheMachine(machine string, moved []string, since time.Time, f gateFacts) machineWord { - w := machineWord{facts: f, on: map[string]string{}} - if !f.judged || f.openErr != nil { - return w - } - var core []string - for _, m := range moved { - if c := coreComponent(m); c == lease.ComponentEngine || c == lease.ComponentNodeTools { - core = append(core, m) - } - } - kept := make([]conditions.Condition, 0, len(f.open)) - for _, c := range f.open { - aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine || - slices.Contains(c.Subject.Also, machine)) - // A directory used as found waits for a person, whatever the send did (novox/hq issue 339). - if !aboutIt || c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindUsedAsFound { - kept = append(kept, c) - continue - } - said := fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary) - parts := strings.Split(c.Subject.ID, ".") - var named []string - for _, m := range moved { - if slices.Contains(parts, m) { - named = append(named, m) - } - } - coreBehind := c.Kind == kindCoreBehind || strings.HasSuffix(c.Key, "."+kindCoreBehind) - switch { - case len(named) > 0: - for _, m := range named { - if _, already := w.on[m]; !already { - w.on[m] = said - } - } - case coreBehind && len(core) > 0: - for _, m := range core { - if _, already := w.on[m]; !already { - w.on[m] = said + " (its settle window runs to the gate's bound)" - } - } - case coreBehind: - // Not what the send moved: said nowhere against it. - case c.Kind == kindMachineUnits: - // **The machine's own failed units** (issue 315): no module places any of them, so nothing a - // send moved is among them — a module's failed unit is that module's own condition. Said - // nowhere against the send. - case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine: - // **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send - // did not move, or the machine cannot reach another that is down. Nothing the send did; the - // judging waits for it rather than putting back a build at the bound. - if w.waiting == "" { - w.waiting = machine + "'s network: " + said - } - default: - if w.whole == "" { - w.whole = machine + " as a whole: " + said - } - } - } - w.facts.open = kept - return w -} - -// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass -// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one. -func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, - running []string, now time.Time) (string, error) { - g := state.Gate - if g == nil { - // Judged from the send: a witness's verdict, a condition, the bound — all counted from when the - // first machine was sent the build. - start := now - if state.FirstAt != nil { - start = *state.FirstAt - } - var machines []string - for _, n := range state.First { - if slices.Contains(running, n) { - machines = append(machines, n) - } - } - g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous, - To: state.Commit, Since: &start} - state.Gate = g - } - pairs := []judged{} - for _, n := range g.Machines { - pairs = append(pairs, judged{module: module, node: n}) - } - return judgeMoves(ctx, open, g, pairs, now) -} - -// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one -// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352). -func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool { - most := g.Since.Add(gateWithin) - return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most) -} - -// judged is one module on one machine, as a gate judges it. -type judged struct{ module, node string } - -// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and -// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is -// wanting, and which modules), or the verdict. Answers the verdict once there is one. -func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) { - if g.Verdict != "" { - return g.Verdict, nil - } - // A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing: - // a module healthy throughout is never held past it (novox/hq ADR 0352). - if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) { - return "", nil - } - for _, c := range g.Carried { - if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) { - pairs = append(pairs, judged{module: c.Module, node: c.Node}) - } - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - facts, err := gatherGateFacts(ctx, open, g.Component) - if err != nil { - return "", err - } - facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf) - facts.waits = gateWaitFacts(ctx, open.inventory, g, pairs, shelf, facts.health) - facts.sent = g.Sent - facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{} - // A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself: - // not another send, and not a judging superseded. - for _, m := range g.Returned { - delete(facts.commits, m) - } - for _, j := range pairs { - if _, read := facts.sentBuilds[j.node]; read { - continue - } - if builds, known, err := open.inventory.SentBuilds(ctx, j.node); err == nil && known { - facts.sentBuilds[j.node] = builds - } - } - // **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the - // walk says it waits there, and its gate starts at the report that says the container replaced. - if busyWait(g, pairs, facts, now) { - return "", nil - } - // **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each - // machine judged, apart from what is wrong with a module there, and never pinned on the module the - // gate happens to be kept on. - byMachine := map[string][]string{} - for _, j := range pairs { - byMachine[j.node] = append(byMachine[j.node], j.module) - } - words := map[string]machineWord{} - for node, moved := range byMachine { - words[node] = aboutTheMachine(node, moved, *g.Since, facts) - } - // **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its - // machines', its passes are counted apart, and the send's verdict is still one. **Every module of a send - // leaves it with a verdict** (issue 318 review): one healthy for the passes the gate asks, after the - // settle time, keeps a pass when another beside it fails; one found broken holds the send's verdict until - // the modules beside it have theirs, or the bound; and at the verdict, every module that did not pass is - // put back with what failed, so none is left on the machine unjudged for other walks to wait on. - worst, why := healthGood, "" - reading := map[string]health{} - waits := map[string]string{} - var modules []string - for _, j := range pairs { - w := words[j.node] - h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts) - if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) { - h, said = healthNotYet, why - } - if h == healthGood || h == healthPerson { - if on, named := w.on[j.module]; named { - h, said = healthNotYet, on - } else if w.whole != "" { - h, said = healthNotYet, w.whole - } else if w.waiting != "" { - h, said = healthWaiting, w.waiting - } - } - if _, seen := reading[j.module]; !seen { - modules = append(modules, j.module) - } - if h == healthSuperseded { - // Decided at once (novox/hq issue 352): nothing of this gate's can be judged on a machine that - // was sent another build of it, and nothing is put back — the later send is what runs there. - g.Failing, g.Last = nil, "" - decide(g, inventory.GateSuperseded, said, now) - return g.Verdict, nil - } - if h == healthBroken && !slices.Contains(g.Broken, j.module) { - g.Broken = append(g.Broken, j.module) - if g.BrokenWhy == "" { - g.BrokenWhy = said - } - } - if slices.Contains(g.Broken, j.module) { - h = healthBroken // found broken once, broken for the rest of the judging - } - if h > reading[j.module] { - reading[j.module] = h - } - if h == healthPerson { - waits[j.module] = joinSaid(waits[j.module], said) - } - if h > worst { - worst, why = h, said - } else if h == worst && h > healthPerson && why == "" { - why = said - } - } - // **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read - // each machine report its send applied, and who holds the controller lease, the earliest its reading counts. - if g.Applied == nil { - g.Applied = map[string]time.Time{} - } - for node := range byMachine { - r, said := facts.reports[node] - if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied && - facts.reportedOn(node, r) { - g.Applied[node] = *r.At - } - } - rules, floors := map[string]gateRule{}, map[string]time.Time{} - for _, m := range modules { - var on []string - for _, j := range pairs { - if j.module == m { - on = append(on, j.node) - } - } - rules[m] = gateRuleOf(m, shelf[m]) - floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder) - } - tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why, - rules: rules, floors: floors}, now) - return g.Verdict, nil -} - -// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading -// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's -// rule and the earliest its reading counts. -type gateJudging struct { - modules []string - reading map[string]health - waits map[string]string - worst health - why string - rules map[string]gateRule - floors map[string]time.Time -} - -// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but -// for the gate it writes. -func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { - modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why - if g.Healthy == nil { - g.Healthy = map[string]int{} - } - if g.HealthyAt == nil { - g.HealthyAt = map[string]time.Time{} - } - g.Waits = nil - var failing []string - for _, m := range modules { - if reading[m] > healthPerson { - failing = append(failing, m) - g.Healthy[m] = 0 - delete(g.HealthyAt, m) - continue - } - if w := waits[m]; w != "" { - if g.Waits == nil { - g.Waits = map[string]string{} - } - g.Waits[m] = w - } - // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the - // controller's and the tool runner's after the witness window, any other module's from the send. - if now.Before(j.floors[m]) { - continue - } - // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more - // often while another module beside it is not yet healthy counts no faster. - if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { - g.Healthy[m]++ - g.HealthyAt[m] = now - } - } - // What the send needs is what its most demanding module needs — the most readings, then the latest - // floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout - // its two minutes (novox/hq ADR 0352). - wants, floor, within, said := 0, time.Time{}, time.Duration(0), "" - for _, m := range modules { - r, f := j.rules[m], j.floors[m] - if r.passes > wants || (r.passes == wants && f.After(floor)) { - wants, within, said = r.passes, r.within, r.said - } - if f.After(floor) { - floor = f - } - } - if len(modules) == 0 { - // A send judged on no machine of its modules (its first machine not running them) has no class to - // be judged by: it is judged as the core, never passed at once without a reading. - wants, floor, within = gatePasses, *g.Since, gateWithin - said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin) - } - // heldLongEnough is a rule's most: within after the send, healthy at every reading since it. - heldLongEnough := func(within time.Duration) bool { - return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within)) - } - g.Wants = wants - // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after - // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). - passedAlone := func(m string) bool { - r := j.rules[m] - return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within))) - } - // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. - fail := func(why string) { - g.Passing, g.Failing = nil, nil - for _, m := range modules { - if passedAlone(m) { - g.Passing = append(g.Passing, m) - } else { - g.Failing = append(g.Failing, m) - } - } - decide(g, inventory.GateFailed, why, now) - } - pastBound := now.Sub(*g.Since) > gateBound - switch { - case worst == healthBroken: - g.Faulted = true - g.Read(now, false, g.BrokenWhy) - var judging []string - for _, m := range modules { - if reading[m] != healthBroken && !passedAlone(m) { - judging = append(judging, m) - } - } - if len(judging) == 0 || pastBound { - fail(g.BrokenWhy) - break - } - // What broke fails the send, and is put back at once by the caller (putBackBroken); what is beside - // it is judged to its own verdict first, within the bound. - g.Passes, g.LastPass, g.Failing = 0, nil, failing - g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", ")) - case worst == healthWaiting: - // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — - // the provider's own condition says what is wrong (ADR 0240 rule 5). - g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting - g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing - g.Read(now, false, why) - case worst == healthNotYet: - g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting - g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing - g.Read(now, false, why) - if pastBound { - fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why)) - } - case now.Before(floor): - // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). - g.HealthySeen = true - // The bound still holds: a module that cannot be read healthy after its floor fails at it. - g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", - floor.UTC().Format(time.RFC3339), said), nil - if pastBound { - fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last)) - } - default: - // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. - g.HealthySeen = true - g.Passes++ - g.LastPass, g.Last, g.Failing = &now, "", nil - g.Read(now, true, "") - if g.Passes >= wants || heldLongEnough(within) { - decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, - now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) - } - } -} - -// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each -// carried move's. Pure. -func judgedCommits(g *inventory.PlanGate, pairs []judged) map[string]string { - out := map[string]string{} - for _, c := range g.Carried { - if c.To != "" { - out[c.Module] = c.To - } - } - if g.To != "" { - for _, j := range pairs { - if _, has := out[j.module]; !has && !slices.ContainsFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == j.module }) { - out[j.module] = g.To - } - } - } - return out -} - -// sentNow is what each machine was just sent, read after a send for the gate to keep (novox/hq issue -// 352): a machine whose send is not on record is left out, and its report is read as before. -func sentNow(ctx context.Context, inv *inventory.Inventory, machines []string) map[string]inventory.SentDeclaration { - out := map[string]inventory.SentDeclaration{} - for _, n := range machines { - if s, found, err := inv.SentTo(ctx, n); err == nil && found { - out[n] = s - } - } - if len(out) == 0 { - return nil - } - return out -} - -// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait -// for a person, never another's (issue 318 review). -func whyFor(g *inventory.PlanGate, module string) string { - why, _, _ := strings.Cut(g.Why, waitsPrefix) - if w := g.Waits[module]; w != "" { - why += waitsPrefix + w - } - return why -} - -const waitsPrefix = "; and it waits for a person: " - -// waitsSaid is the waits for a person a passing gate carries, as its verdict says them. -func waitsSaid(waits map[string]string) string { - if len(waits) == 0 { - return "" - } - modules := make([]string, 0, len(waits)) - for m := range waits { - modules = append(modules, m) - } - sort.Strings(modules) - var said []string - for _, m := range modules { - said = append(said, waits[m]) - } - return waitsPrefix + strings.Join(said, "; ") -} - -func joinSaid(a, b string) string { - switch { - case a == "": - return b - case b == "" || strings.Contains(a, b): - return a - } - return a + "; " + b -} - -// movesAddingGroups is, per module a gate judges, whether its move puts an account in a group the build it -// moved from did not: read from the builds' manifests. A module whose move is not known adds none. -func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged, - shelf map[string]catalogue.Manifest) map[string]bool { - out := map[string]bool{} - for _, j := range pairs { - if _, done := out[j.module]; done { - continue - } - from, to := g.From, g.To - for _, c := range g.Carried { - if c.Module == j.module { - from, to = c.From, c.To - break - } - } - target, found, err := inv.ManifestAt(ctx, j.module, to) - if err != nil || !found { - target = shelf[j.module] - } - // What it moved from is not known — no build named (a plan's own module whose previous build was not - // recorded), or no manifest kept for it: no wait is excused, rather than one the move did not bring. - if from == "" { - out[j.module] = false - continue - } - before, had, err := inv.ManifestAt(ctx, j.module, from) - if err != nil || !had { - out[j.module] = false - continue - } - out[j.module] = addsAccountGroups(before, had, target) - } - return out -} - -// gateWaitFacts reads what checks the waits for the operator of the modules a gate judges (novox/hq ADR 0283): the -// manifest of the build judged — the one it moves to, else the catalogue's — and the secrets given on each machine -// that says a module of them waits. -func gateWaitFacts(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged, - shelf map[string]catalogue.Manifest, health map[string]inventory.NodeHealth) operatorWaitFacts { - var f operatorWaitFacts - judgedManifests := map[string]catalogue.Manifest{} - byMachine := map[string][]string{} - for _, j := range pairs { - waiting := false - for _, r := range health[j.node].Resources { - if r.Module == j.module && r.State == link.StateWaiting { - waiting = true - } - } - if !waiting { - continue - } - byMachine[j.node] = append(byMachine[j.node], j.module) - if _, done := judgedManifests[j.module]; done { - continue - } - to := g.To - for _, c := range g.Carried { - if c.Module == j.module { - to = c.To - break - } - } - if m, found, err := inv.ManifestAt(ctx, j.module, to); err == nil && found { - judgedManifests[j.module] = m - } else if m, known := shelf[j.module]; known { - judgedManifests[j.module] = m - } - } - for machine, modules := range byMachine { - readWaitFacts(ctx, inv, machine, modules, judgedManifests, &f) - } - return f -} - -// decide sets a gate's verdict. -func decide(g *inventory.PlanGate, verdict, why string, now time.Time) { - g.Verdict, g.Why, g.JudgedAt = verdict, why, &now - g.Took = now.Sub(*g.Since).Round(time.Second).String() -} - -// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it. -func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) { - g := state.Gate - err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, - Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, - Verdict: inventory.GatePassed, Why: whyFor(g, module), Component: g.Component, JudgingFrom: g.Since}) - if err != nil && state.Build != "" { - fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err) - } - passCarried(ctx, open, p, g, module) - fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module, - strings.Join(g.Machines, ", "), g.Why) - if module == catalogue.ControllerSeatName { - carryUserList(ctx, open, p) - } -} - -// carryUserList sends the machine holding the bus the user list a new controller composes, once that -// controller passed its gate (ADR 0236). The controller's own grants travel in that list, and the old -// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new -// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR -// 0221): then it is said, as a push would say it. -func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) { - holder, behind, err := brokerBehind(ctx, open, nil) - if err != nil || holder == "" || !behind { - if err != nil { - fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err) - } - return - } - held, err := heldMachines(ctx, open, []string{holder}) - if err != nil { - fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err) - return - } - if why, isHeld := held[holder]; isHeld { - fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+ - "carries it\n", p.ID, holder, strings.Join(why, "; "), holder) - return - } - if _, err := sendRollout(ctx, open, []string{holder}); err != nil { - fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err) - return - } - fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder) -} - -// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the -// machines it was judged on — once per build, said as a condition and an event. The plan is saved -// before the send: a controller that is itself the build being put back does not outlive it. -func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, - machines []string, why string) { - inv := open.inventory - now := time.Now().UTC() - if state.Gate == nil { - state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, - From: state.Previous, To: state.Commit, Since: state.FirstAt} - } - g := state.Gate - if g.Verdict == "" { - if g.Since == nil { - g.Since = &now - } - decide(g, inventory.GateFailed, why, now) - } - if len(g.Machines) == 0 { - g.Machines = machines - } - state.Why = "failed its gate: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why) - - verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous, - Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why, - Component: g.Component, JudgingFrom: g.Since} - // **A send that changed nothing of the module there is no verdict on its build** (novox/hq issue - // 280). The machine already ran this build, or one that made the same artifacts from the same - // manifest: whatever the gate found wanting, this build did not bring it, and there is nothing to - // put back. On 2026-10-06 such a module was marked failed, and the rollback looked for an earlier - // build of the very commit it had failed — "no build kept" — while the build it had run before was - // kept all along. Said, left as it is, never marked. - if unchangedBy(ctx, inv, module, state.Previous, state.Commit) { - g.Rollback = gateUnchanged - state.Why = "stopped with its send; the send changed nothing of it: " + g.Why - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was left as it was — %s already ran "+ - "%s %s, or a build identical to it, before the send, so nothing of it moved and nothing is put back", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module, strings.Join(g.Machines, ", "), module, short(state.Commit)) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return - } - if state.Build == "" { - // A plan from before builds were asked by id: nothing to mark, so nothing is put back by the - // mesh — said, for a person. - g.Rollback = inventory.NotRolledBack - p.Note += "; not put back: the plan does not know which build it sent" - sayRollback(ctx, open, module, g, "") - return - } - if err := inv.RecordGate(ctx, verdict); err != nil { - if errors.Is(err, inventory.ErrGateKept) { - // Already judged and acted on, by this controller before a restart or by another: never twice. - if kept, found, _ := inv.GateOf(ctx, state.Build); found { - g.Rollback = kept.Rollback - } - p.Note += "; its rollback was already made once and is not made again" - return - } - g.Rollback = inventory.NotRolledBack - p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error() - sayRollback(ctx, open, module, g, "") - return - } - // The previous build: the one the first machine ran, from the build records. - notBack := func(why string) { - g.Rollback = inventory.NotRolledBack - p.Note += "; NOT put back: " + why - if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err) - } - sayRollback(ctx, open, module, g, why) - } - if state.Previous == "" { - // A first build there: nothing ran before it, so nothing can be put back, and the machine is left - // with it — said as that, not as a build the mesh lost. - notBack(fmt.Sprintf("this is the first build of %s that %s was sent, or what it was sent before is not known: "+ - "there is no earlier build there to put back, so it is left with this one — `unassign` takes it off, "+ - "a newer merge replaces it", module, strings.Join(g.Machines, ", "))) - return - } - failed, _, err := inv.BuildByID(ctx, state.Build) - if err != nil { - notBack("the failed build's record cannot be read: " + err.Error()) - return - } - previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed) - if err != nil { - notBack("the build records cannot be read: " + err.Error()) - return - } - if !found { - notBack(fmt.Sprintf("no build of %s from %s, asked before the failed one, is among the %d newest kept to put "+ - "back", module, short(state.Previous), inventory.KeptBuilds)) - return - } - if g.Component == lease.ComponentController { - // **The controller is never put back to a build older than the store's schema** (novox/hq issue - // 352): the build before it carries fewer migrations than the failed one applied, starts behind - // its own records, and judges the next gate with what it can read. The current build is kept and - // the condition says so; a person decides. - if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { - notBack(why) - return - } - } - if err := inv.RestoreModule(ctx, previous); err != nil { - notBack(err.Error()) - return - } - g.Rollback = inventory.RollingBack - if err := inv.SavePlan(ctx, p); err != nil { - fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err) - } - // Put back with the rest of its send, in one send per machine (issue 281). - if b, batched := ctx.Value(rollbacksKey{}).(*rollbacks); batched { - b.pending = append(b.pending, pendingRollback{module: module, state: state, g: g, previous: previous}) - b.modules[module] = true - for _, n := range g.Machines { - if !slices.Contains(b.machines, n) { - b.machines = append(b.machines, n) - } - } - return - } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines) - if err != nil { - notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ - "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) - return - } - g.Rollback = inventory.RolledBack - p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", ")) - if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err) - } - sayRollback(ctx, open, module, g, "") -} - -// controllerSchemaAllows says the store's schema lets this build of the controller be put back: the -// build recorded, when it served, a reach at or past the highest migration the store has applied. One -// that never recorded a reach is not proved safe, and is refused as such (novox/hq issue 352). Why -// says what is kept and why when it is not. -func controllerSchemaAllows(ctx context.Context, inv *inventory.Inventory, previous inventory.Build) (string, bool) { - applied, err := inv.SchemaApplied(ctx) - if err != nil { - return "what the store's schema reaches cannot be read, so whether the build before it can read it is not " + - "known; the current build is kept: " + err.Error(), false - } - build := versionOfBuild(previous) - if build == "" { - return fmt.Sprintf("the build before it (%s) names no bundle to know it by, so whether it can read the store's "+ - "schema (migration %04d) is not known; the current build is kept, and a person decides", short(previous.Commit), applied), false - } - reach, known, err := inv.SchemaReachOf(ctx, build) - if err != nil { - return "what the build before it knows of the store's schema cannot be read; the current build is kept: " + err.Error(), false - } - if !known { - return fmt.Sprintf("the build before it (%s, %s) never recorded how far it reads the store's schema — a "+ - "controller records that when it serves — so it is not proved to read migration %04d, which the store "+ - "has applied; a controller older than its store starts behind its own records and judges with what it "+ - "can read, so the current build is kept, and a person decides", short(previous.Commit), build, applied), false - } - if reach < applied { - return fmt.Sprintf("the build before it (%s, %s) reads the store's schema up to migration %04d, and the store "+ - "is at %04d: a controller older than its store starts behind its own records and judges with what it "+ - "can read, so the current build is kept, and a person decides", short(previous.Commit), build, reach, applied), false - } - return "", true -} - -// versionOfBuild is the version a build's bundle is delivered as — its archive's digest, short, as the -// catalogue names it (`${version}`) — read from the build's artifacts; empty when none is a bundle. -func versionOfBuild(b inventory.Build) string { - for _, a := range b.Made { - if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactArchive { - continue - } - if _, hex, found := strings.Cut(a.Reference, "sha256:"); found && len(hex) >= 12 { - return hex[:12] - } - } - return "" -} - -// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one -// send judges what it moved as one, and what it found wanting goes back in one send per machine — not -// in a send for each module, which is the churn that failed the gate in the first place. -type rollbacks struct { - modules map[string]bool - machines []string - pending []pendingRollback -} - -type pendingRollback struct { - module string - state *inventory.PlanModule - g *inventory.PlanGate - previous inventory.Build -} - -type rollbacksKey struct{} - -// batchingRollbacks is a context under which gateFailed registers what it puts back and leaves the send -// to sendRollbacks. -func batchingRollbacks(ctx context.Context) (context.Context, *rollbacks) { - b := &rollbacks{modules: map[string]bool{}} - return context.WithValue(ctx, rollbacksKey{}, b), b -} - -// sendRollbacks sends what a failed send put back, once to each machine, and says each module's rollback. -func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *rollbacks) { - if len(b.pending) == 0 { - return - } - inv := open.inventory - sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines) - var back []string - for _, r := range b.pending { - if err != nil { - why := fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ - "sends it", short(r.previous.Commit), strings.Join(r.g.Machines, ", "), err, firstOf(r.g.Machines)) - r.g.Rollback = inventory.NotRolledBack - if err := inv.SetRollback(ctx, r.state.Build, inventory.NotRolledBack, r.g.Why+"; not put back: "+why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err) - } - sayRollback(ctx, open, r.module, r.g, why) - continue - } - r.g.Rollback = inventory.RolledBack - back = append(back, r.module+" to "+short(r.previous.Commit)) - if err := inv.SetRollback(ctx, r.state.Build, inventory.RolledBack, r.g.Why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err) - } - sayRollback(ctx, open, r.module, r.g, "") - } - if err != nil { - p.Note += fmt.Sprintf("; NOT put back: sending %s was refused: %v", strings.Join(b.machines, ", "), err) - return - } - p.Note += fmt.Sprintf("; put back %s on %s, in one send", strings.Join(back, ", "), strings.Join(sent, ", ")) -} - -// gateUnchanged is a failed gate's word on a module its send changed nothing of (novox/hq issue 281): -// left as it was, never marked failed, and so no verdict on its build — `plans retry` asks it again. -const gateUnchanged = "unchanged" - -// unchangedBy says whether a send moving a module from one build to another changed nothing of it: the -// same commit, builds made from the same source, or builds that made the same artifacts from the same -// manifest. Not known is changed. -func unchangedBy(ctx context.Context, inv *inventory.Inventory, module, from, to string) bool { - if from == "" || to == "" { - return false - } - if sameCommit(from, to) { - return true - } - // Made from the same source (issue 280), or the same artifacts from the same manifest. - f := moveFacts{} - var err error - if f.srcs, err = inv.SourceFingerprints(ctx); err != nil { - return false - } - if f.fps, err = inv.Fingerprints(ctx); err != nil { - return false - } - return f.identical(module, from, to) -} - -// rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events. -type rolledBackEvent struct { - Event string `json:"event"` - At time.Time `json:"at"` - Module string `json:"module"` - Component string `json:"component,omitempty"` - Machines []string `json:"machines"` - From string `json:"from,omitempty"` - To string `json:"to,omitempty"` - Why string `json:"why"` - // Rollback is rolled-back, or not-rolled-back with NotWhy. - Rollback string `json:"rollback"` - NotWhy string `json:"not_why,omitempty"` - Show string `json:"show"` -} - -// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event. -func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) { - o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From) - fmt.Println(o.Summary) - d := doctorFrom - if d == nil { - return - } - if d.keeper != nil { - o.Source = gateProbe - if _, err := d.keeper.Observe(ctx, o); err != nil { - fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err) - } - } - if d.teller == nil { - return - } - body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module, - Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback, - NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()}) - if err != nil { - return - } - saying, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil { - fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err) - } -} - -// gateObservation is a failed gate as a condition: `core...rolled-back` for the -// core (to-be 45 §2), `build...rolled-back` for any other module; `rollback-failed` -// when it could not be put back. Urgent for the core and for a build left in place; a warning for a -// module put back, which runs what it ran before. -func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation { - machine := strings.Join(machines, ",") - o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack, - Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe, - Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed), - machine, short(previous), why)} - if component != "" { - o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent - } - if len(machines) > 1 { - o.Also = machines[1:] - } - if rollback != inventory.RolledBack && rollback != inventory.RollingBack { - o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent - o.Resolver = conditions.ResolverOperator - o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed), - machine, why, orNone(notWhy)) - } - return o -} - -// probeGates is DG: no build that failed its gate is left without its condition, and no witness's -// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build -// that passes clears it. Each core component a machine's witness says it put back is `core.. -// .rolled-back`, urgent, until the machine's reports stop saying it. -func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - latest, err := d.open.inventory.LatestGates(ctx) - if err != nil { - return nil, err - } - var out []conditions.Observation - for _, v := range latest { - if v.Verdict != inventory.GateFailed { - continue - } - notWhy := "" - if v.Rollback == inventory.NotRolledBack { - notWhy = v.Why - } - out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous)) - } - for node, list := range witnessed.all() { - for _, r := range list { - out = append(out, witnessObservation(node, r)) - } - } - // A release held after a failed one, while builds still wait for a gate (ADR 0236). - out = append(out, backlogObservation()...) - // And a rest that went silent (novox/hq ADR 0353 §4). - silent, err := silentObservation(ctx, d.open.inventory) - if err != nil { - return nil, err - } - out = append(out, silent...) - return sortedFound(dedupeObservations(out)), nil -} - -// dedupeObservations keeps one observation per key, the first. -func dedupeObservations(list []conditions.Observation) []conditions.Observation { - seen := map[string]bool{} - var out []conditions.Observation - for _, o := range list { - if seen[o.Key()] { - continue - } - seen[o.Key()] = true - out = append(out, o) - } - return out -} - -// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules' -// tools are served where. A variable so a test needs no runtime. -var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) { - inbox := conn.NewRespInbox() - sub, err := conn.SubscribeSync(inbox) - if err != nil { - return nil, err - } - defer func() { _ = sub.Unsubscribe() }() - if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil { - return nil, fmt.Errorf("asking the bus who serves what: %w", err) - } - out := map[string]served{} - add := func(node string) served { - s, ok := out[node] - if !ok { - s = served{tools: map[string]bool{}} - } - return s - } - deadline := time.Now().Add(discoveryPatience) - for time.Now().Before(deadline) { - wait, cancel := context.WithTimeout(ctx, discoveryQuiet) - msg, err := sub.NextMsgWithContext(wait) - cancel() - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - break - } - var info micro.Info - if json.Unmarshal(msg.Data, &info) != nil { - continue - } - if info.Name == broker.RuntimeModule { - node := info.Metadata["node"] - if node == "" { - node = info.ID - } - s := add(node) - s.runtime = true - out[node] = s - } - for _, e := range info.Endpoints { - if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" { - continue - } - node := e.Metadata["node"] - if node == "" { - node = info.ID - } - s := add(node) - s.tools[e.Metadata["module"]] = true - out[node] = s - } - } - return out, nil -} - -// gateLines is a plan's gates as `plans ` says them: the rollout's record. -func gateLine(g *inventory.PlanGate) string { - if g == nil { - return "" - } - what := "judging" - switch g.Verdict { - case inventory.GatePassed: - what = "passed" - case inventory.GateFailed: - what = "FAILED" - } - switch { - case g.Verdict == "" && g.Pending: - what = "kept, not yet sent" // novox/hq ADR 0353 §8 - case g.Verdict == "" && g.Silent: - what = "silent on its send, left" // ADR 0353 §4 - } - if g.Verdict == "" && g.Waiting() { - what = "waits for a busy container on " + g.Busy.Machine - } - line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what) - if g.Component != "" { - line += " (core: " + g.Component + ")" - } - if g.From != "" || g.To != "" { - line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To)) - } - if g.Took != "" { - line += ", after " + g.Took - } - if g.Why != "" { - line += ": " + g.Why - } else if g.Last != "" { - line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last) - } else if g.Verdict == "" { - line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g)) - } - if g.Verdict == "" && len(g.Waits) > 0 { - line += waitsSaid(g.Waits) - } - if len(g.Passing) > 0 { - line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ") - } - if g.Rollback != "" { - line += "; " + g.Rollback - } - // Before the send it judges (novox/hq issue 490). - if said := grantsSaid(g.Grants); said != "" { - line += "; " + said - } - return line -} - -// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract): -// `core...`, urgent for rolled-back, not-reversible, restore-failed and -// halted, a warning for nothing-to-restore and unwitnessed. -func witnessObservation(node string, r lease.Rollback) conditions.Observation { - severity := conditions.Warning - if lease.Urgent(r.Outcome) { - severity = conditions.Urgent - } - outcome := r.Outcome - if outcome == "" { - outcome = lease.OutcomeRolledBack - } - what := map[string]string{ - lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)), - lease.OutcomeNotReversible: "and left it: it is not reversible", - lease.OutcomeNothingToRestore: "and had nothing to put back", - lease.OutcomeRestoreFailed: "and could not put the previous build back", - lease.OutcomeUnwitnessed: "and could not judge it at all", - lease.OutcomeHalted: "and gave up: the build before it fails too", - }[outcome] - return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack, - Token: outcome, Machine: node, Severity: severity, - Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component, - short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)} -} - -// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said. -func gateWanted(g *inventory.PlanGate) int { - if g.Wants > 0 { - return g.Wants - } - return gatePasses -} diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go deleted file mode 100644 index ca5dbb9f..00000000 --- a/cmd/mesh-controller/gate_followup_test.go +++ /dev/null @@ -1,468 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "reflect" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a -// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own -// wait, and a provider waiting for a login says who waits on it. - -// withGateBounds sets the gate's bounds for one test. -func withGateBounds(t *testing.T, within, every, bound time.Duration) { - t.Helper() - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery, gateBound = within, every, bound - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) -} - -// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says -// of its modules' resources. -func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts { - return func() gateFacts { - now := time.Now() - at := now - return gateFacts{now: now, - reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, - engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, - served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}}, - health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}}, - } - } -} - -func healthyContainer(module string) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateHealthy} -} - -// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node -// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, -// counted over the gate's spacing — before it says its own. The broken one alone is put back. -func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - open := aMesh(t) - ctx := t.Context() - withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second) - since := time.Now().Add(-time.Second) - base := factsOn(t, "laptop", since, healthyContainer("app")) - wasGather := gatherGateFacts - t.Cleanup(func() { gatherGateFacts = wasGather }) - gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { - f := base() - f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack, - At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}} - return f, nil - } - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} - pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}} - judgings := 0 - for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); { - if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil { - t.Fatal(err) - } - judgings++ - time.Sleep(30 * time.Millisecond) - } - if g.Verdict != inventory.GateFailed || judgings < gatePasses { - t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict, - judgings, g) - } - if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) { - t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing) - } - if !strings.Contains(g.Why, "witness") { - t.Errorf("the verdict does not say what broke: %q", g.Why) - } -} - -// broker_runtime is the node tools' module name, a core component a witness judges. -const broker_runtime = "node-tools" - -// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick -// while a module beside it is not yet healthy counts the healthy one once. -func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - withGateBounds(t, time.Hour, time.Hour, time.Hour) - since := time.Now().Add(-time.Minute) - base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late", - Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}) - wasGather := gatherGateFacts - t.Cleanup(func() { gatherGateFacts = wasGather }) - gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil } - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} - pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}} - now := time.Now() - for i := 0; i < 3; i++ { - if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil { - t.Fatal(err) - } - } - if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 { - t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy) - } -} - -// **A carried build's pass carries its own wait, never another's** (review (e)). -func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - since := time.Now().Add(-time.Minute) - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed, - Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}), - Waits: map[string]string{"late": "relogin needed on laptop: late waits"}, - Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"}, - {Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}} - passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "") - app, _, _ := inv.GateOf(ctx, "build-app-c2") - late, _, _ := inv.GateOf(ctx, "build-late-c2") - if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed { - t.Errorf("app's pass: %+v; want it without late's wait", app) - } - if !strings.Contains(late.Why, "relogin needed on laptop") { - t.Errorf("late's pass: %+v; want its own wait", late) - } -} - -// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send -// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the -// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it. -func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // `late` waits on anchor for a gate: c1 sent, c2 registered and built. - for _, b := range []inventory.Build{ - {ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit}) - b.Manifest = manifest - b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit, - Asked: b.Asked}); err != nil { - t.Fatal(err) - } - if b.Commit == "l1" { - if _, err := inv.Assign(ctx, "anchor", "late"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1", - "late": "l1"}); err != nil { - t.Fatal(err) - } - } - } - wasMoves := machineMoves - t.Cleanup(func() { machineMoves = wasMoves }) - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - gather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := gather(ctx, open, component) - f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ - healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late", - State: link.StateUnhealthy, Reason: "down"}}}} - return f, err - } - gateEvery, gateBound = 0, 400*time.Millisecond - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, g.open) - if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone { - break - } - time.Sleep(30 * time.Millisecond) - } - p := g.plan(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note) - } - if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed || - !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) - } - if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed { - t.Fatal("late's build is not marked failed at its gate") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" { - t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit, - current["late"].Commit) - } -} - -// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that -// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent. -func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - k := conditionsFrom - register(t, open, catalogue.Manifest{Module: "db", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) - for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", - Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { - t.Fatal(err) - } - at := h0 - say := func(machine string, rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator", - Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"} - unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service", - Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} - shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", - State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} - for look := 0; look < 2; look++ { - say("anchor", account, unit) - say("laptop", shop) - } - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var keys []string - var c conditions.Condition - for _, x := range list { - keys = append(keys, x.Key) - if x.Key == "module.db.anchor.relogin-needed" { - c = x - } - } - if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) { - t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys) - } - if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") { - t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said) - } -} - -// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back -// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node -// tools are marked and put back in the very judging that found them broken — their registered build is the -// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. -func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute) - build := func(module, commit string, asked time.Time) { - manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"app", broker_runtime} { - build(m, "c1", old) - if _, err := inv.Assign(ctx, "laptop", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil { - t.Fatal(err) - } - build("app", "c2", recent) - build(broker_runtime, "c2", recent) - - wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - }) - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, _ := open.inventory.Assigned(ctx, node) - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil } - var sends []string - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit} - sends = append(sends, node+":"+carried[broker_runtime]) - digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n)) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - var seen []string // the node tools' registered build at each judging - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - now := time.Now() - f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, _ := open.inventory.LastReports(ctx) - for _, r := range reports { - f.reports[r.Node] = r - } - current, _ := open.inventory.CurrentBuilds(ctx) - seen = append(seen, current[broker_runtime].Commit) - app := healthyContainer("app") - if current[broker_runtime].Commit == "c2" { - // The witness reverted the node tools; app cannot reach them yet. - f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack, - From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}} - app.State, app.Reason = link.StateUnhealthy, "down" - } - f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}} - f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}} - return f, nil - } - withGateBounds(t, time.Hour, 0, 10*time.Second) - - release := func() inventory.Plan { - t.Helper() - plans, _ := inv.RecentPlans(ctx, 10) - for _, p := range plans { - if p.Release != nil { - return p - } - } - t.Fatal("no walk") - return inventory.Plan{} - } - // Judged until the first judging has found the node tools broken, and one more. - for i := 0; i < 20 && len(seen) < 2; i++ { - advancePlans(ctx, open) - } - if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { - t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen) - } - if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed { - t.Fatal("the node tools' build is not marked failed at once") - } - if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" { - t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate) - } - if !slices.Contains(sends, "laptop:c1") { - t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends) - } - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, open) - if release().State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := release() - if p.State != inventory.PlanFailed { - t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v; want its own pass", v) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit) - } -} - -// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous -// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group. -func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - record := func(commit string, m catalogue.Manifest) { - raw, _ := json.Marshal(m) - at := time.Now().Add(-time.Hour) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-lights-" + commit, Module: "lights", Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/lights", Manifest: raw, Asked: at, At: at}); err != nil { - t.Fatal(err) - } - } - record("c1", catalogue.Manifest{Module: "lights"}) - record("c2", catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", - "name": "operator", "groups": []any{"lights"}}}}) - pairs := []judged{{module: "lights", node: "laptop"}} - shelf := map[string]catalogue.Manifest{} - for _, c := range []struct { - from string - want bool - }{{"c1", true}, {"", false}, {"c0-never-built", false}} { - g := &inventory.PlanGate{From: c.from, To: "c2"} - if got := movesAddingGroups(ctx, inv, g, pairs, shelf)["lights"]; got != c.want { - t.Errorf("a move from %q adds a group: %v; want %v", c.from, got, c.want) - } - } -} diff --git a/cmd/mesh-controller/gate_integration_test.go b/cmd/mesh-controller/gate_integration_test.go deleted file mode 100644 index f0357293..00000000 --- a/cmd/mesh-controller/gate_integration_test.go +++ /dev/null @@ -1,486 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -func aGateMesh(t *testing.T) *gateMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - g := &gateMesh{open: open, health: map[string]health{}} - // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. - wasGated := gateFirstMachine - gateFirstMachine = true - t.Cleanup(func() { gateFirstMachine = wasGated }) - g.keeper, _ = withConditionsInMemory(t) - g.told = &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told} - t.Cleanup(func() { doctorFrom = was }) - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - register("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - register("c2", time.Now().Add(-time.Minute)) - - // Every send: recorded with the build the module is at, applied and reported by the machine. - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - g.sent = append(g.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - // What a judging reads: the store's reports, and a health the test says per machine. - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if g.health[r.Node] == healthBroken { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - for node, h := range g.health { - if h == healthNotYet { - // Not reported on the send: neither the send made last, nor the gate's own (issue 352). - f.rolledBack[node] = nil - r := f.reports[node] - r.Current, r.Declared, r.ReportedSequence = false, "", 0 - f.reports[node] = r - } - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return g -} - -func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - - advancePlans(ctx, g.open) // the first machine is sent the new build - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v, not the first machine alone", g.sent) - } - if p := g.plan(t); p.Modules["app"].Previous != "c1" { - t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"]) - } - - g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging - gateEvery = 0 - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed || - gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } - if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" { - t.Fatalf("anchor was last sent %v, not the previous build", sent) - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" { - t.Fatalf("laptop was sent the failed build: %v", sent) - } - if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed { - t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err) - } - - // Said: a condition for the operator, and the event. - open, err := g.keeper.Open(ctx) - if err != nil { - t.Fatal(err) - } - var key string - for _, c := range open { - if c.Kind == kindRolledBack { - key = c.Key - } - } - if key != "build.app.anchor.rolled-back" { - t.Fatalf("no rolled-back condition for app on anchor: %+v", open) - } - saidIt := false - for _, e := range g.told.Said() { - saidIt = saidIt || e.Event == link.KeyRolledBack - } - if !saidIt { - t.Fatalf("the rollback was not said as an event: %+v", g.told.Said()) - } - - // Never again: more passes send nothing, a second judging rolls nothing back, and the failed build - // heard again is not registered. - advancePlans(ctx, g.open) - state := p.Modules["app"] - gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again") - if len(g.sent) != 2 { - t.Fatalf("sent again after the rollback: %v", g.sent) - } - _, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app", - Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}), - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}) - if err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("the failed build was registered again: %v", err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module moved to %s", current["app"].Commit) - } - if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("a plan stopped at a failed gate was retried: %v", err) - } - - // The probe keeps the condition while the newest verdict is the failure. - obs, err := probeGates(ctx, doctorFrom) - if err != nil || len(obs) != 1 || obs[0].Key() != key { - t.Fatalf("the gate's probe found %+v %v", obs, err) - } -} - -func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", g.sent) - } - gate := p.Modules["app"].Gate - if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a passing build left a condition: %+v %v", obs, err) - } -} - -func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { - g := aGateMesh(t) - gateFirstMachine = false - ctx := t.Context() - judged := false - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - judged = true - return wasGather(ctx, open, component) - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { - t.Fatalf("sent %v, not both machines in one send", g.sent) - } - if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { - t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a rolled build left a condition: %+v %v", obs, err) - } -} - -func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateWithin = time.Hour // a core module could not pass on fewer readings for an hour - advancePlans(ctx, g.open) // the first machine is sent the new build - advancePlans(ctx, g.open) // one healthy judging - p := g.plan(t) - gate := p.Modules["app"].Gate - if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { - t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) - } - if !strings.Contains(gate.Why, "healthy 1 times") { - t.Errorf("the verdict says %q", gate.Why) - } -} - -func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - advancePlans(ctx, g.open) - g.health["anchor"] = healthNotYet - gateEvery = 0 - advancePlans(ctx, g.open) - if p := g.plan(t); !p.Open() || len(g.sent) != 1 { - t.Fatalf("judged before the bound: %s %v", p.State, g.sent) - } - gateBound = -time.Second - advancePlans(ctx, g.open) - p := g.plan(t) - if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || - !strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s, its gate %+v", p.State, gate) - } -} - -func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}, - Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}} - if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil { - t.Fatal(err) - } - u, err := inv.UpgradeOf(ctx, "nats") - if err != nil || u.RollOut || u.From != catalogue.FromBus { - t.Fatalf("the bus's policy is %+v %v", u, err) - } - if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) { - t.Fatalf("a person rolled the bus out: %v", err) - } - var sent [][]string - was := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = was }) - now := time.Now().UTC() - plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now, - State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}}, - Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 { - t.Fatalf("a plan sent the bus: %s %v", p.State, sent) - } - held, err := busHeld(ctx, inv, []string{"anchor", "laptop"}) - if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { - t.Fatalf("a push may send the bus's machine: %v %v", held, err) - } - // Nor may any other send — a plan's for another module on that machine carried the bus with it. - if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { - t.Fatalf("a send to the bus's machine was not refused: %v", err) - } - // A rebuild that made the same artifacts is no move. - for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { - b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} - b.Asked, b.At = time.Now(), time.Now() - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { - t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), - At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", - Reference: "registry/nats@sha256:new"}}}); err != nil { - t.Fatal(err) - } - // The planned step refuses to start without its word on reversibility, and without a snapshot taken - // first by the bus machine's backup holder. - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { - t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) - } - wasSnapshot := takeBusSnapshot - t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) - takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || - !strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 { - t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent) - } - takeBusSnapshot = func(_ context.Context, module, node string) (string, error) { - return node + "'s restore point of " + module, nil - } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { - t.Fatalf("the step sent %v, not the bus's machine", sent) - } - s, found, err := inv.LatestBusStep(ctx) - if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil || - !reflect.DeepEqual(s.Machines, []string{"anchor"}) { - t.Fatalf("the step is %+v %v %v", s, found, err) - } -} - -func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - for _, name := range []string{"gone", "kept"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef", - Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"}, - Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - if len(*asked) != 0 { - t.Fatalf("a deleted module was asked to build: %v", *asked) - } - if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a merge that only deleted a module made a plan: %+v", plans) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if _, still := shelf["gone"]; still { - t.Fatal("a deleted module nothing holds was not forgotten") - } - - // Without the announcer saying what went: the build finds no manifest, and the plan goes on. - asked0 := time.Now().UTC().Add(-time.Minute) - plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0, - State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}}, - Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-k", Module: "kept", Repository: "novox/mesh-catalog", - At: time.Now().UTC(), Asked: asked0, Failed: "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so " + - "there is nothing saying what it is: open …/module.json: no such file or directory"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - p, _ := inv.PlanByID(ctx, "plan-deleted") - if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted { - t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"]) - } -} - -func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { - open := aMesh(t) - d := &doctor{open: open} - at := time.Now().UTC() - witnessed.heard("control", []lease.Rollback{ - {Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack, - Why: "the new controller did not take the lease within 60s", At: at}, - {Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at}, - }, at) - t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) }) - obs, err := probeGates(t.Context(), d) - if err != nil { - t.Fatal(err) - } - got := map[string]conditions.Severity{} - for _, o := range obs { - got[o.Key()] = o.Severity - } - want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent, - "core.node-tools.control.unwitnessed": conditions.Warning} - if !reflect.DeepEqual(got, want) { - t.Fatalf("the witness's verdicts are %v", got) - } - witnessed.heard("control", nil, time.Now()) - if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 { - t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) - } -} diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go deleted file mode 100644 index b1b1033e..00000000 --- a/cmd/mesh-controller/gate_rule_test.go +++ /dev/null @@ -1,347 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass -// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's -// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so. - -// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. -func withTheGatesTimes(t *testing.T) { - t.Helper() - wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow - wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles - gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second - gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false - t.Cleanup(func() { - gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow - gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles - }) -} - -func TestEachModuleHasItsGateRule(t *testing.T) { - withTheGatesTimes(t) - for _, c := range []struct { - module string - claims []catalogue.Claim - passes int - within time.Duration - witnessed bool - said string - }{ - {"mesh-controller", nil, 1, 0, true, "the controller's rule"}, - {"node-tools", nil, 1, 0, true, "the tool runner's rule"}, - {catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"}, - {"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"}, - {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"}, - {"nats", nil, 3, 2 * time.Minute, false, "the core's rule"}, - {"app", nil, 1, 0, false, "a leaf module's rule"}, - } { - r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { - t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within, - c.witnessed, c.said) - } - } -} - -// Every module that was core stays core for its delivery budget and its walk: only its readings change. -func TestTheCoreStaysTheCore(t *testing.T) { - for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} { - // The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 §3, issue 1210). - claims := []catalogue.Claim(nil) - if m == catalogue.DeliverySeat { - claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} - } - if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) { - t.Errorf("%s is no longer judged as the core for its class", m) - } - } - if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) { - t.Error("the resolver's holder is no longer judged as the core for its class") - } -} - -// gateOn judges one module on one machine, healthy at every reading. -type gateOn struct { - module, node string - g *inventory.PlanGate - holder *lease.Holder -} - -func aGateOn(module, node string, since time.Time) *gateOn { - return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since, - Applied: map[string]time.Time{}}} -} - -func (o *gateOn) read(at time.Time, h health, why string) { - m := catalogue.Manifest{Module: o.module} - if o.module == catalogue.DeliverySeat { // today's module, which claims the seat - m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} - } - r := gateRuleOf(o.module, m) - floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder) - tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h}, - worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at) -} - -func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) { - withTheGatesTimes(t) - sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) - o := aGateOn("mesh-controller", "novox", sent) - o.g.Applied["novox"] = sent.Add(15 * time.Second) - o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)} - - // Ready, but inside the witness window, which opened before it took the lease: not counted. - for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} { - o.read(sent.Add(after), healthGood, "") - if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") { - t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g) - } - } - o.read(sent.Add(80*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 { - t.Fatalf("after the witness window: %+v; want passed on one reading", o.g) - } - if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") || - !strings.Contains(o.g.Why, "ADR 0351") { - t.Errorf("the verdict says %q; want the rule it applied", o.g.Why) - } -} - -// A controller that starts again takes the lease again: its witness window starts again, and so does the -// gate's wait. -func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - o := aGateOn("mesh-controller", "novox", sent) - o.g.Applied["novox"] = sent.Add(10 * time.Second) - o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)} - o.read(sent.Add(90*time.Second), healthGood, "") - if o.g.Verdict != "" { - t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g) - } - o.read(sent.Add(130*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed { - t.Fatalf("not passed after the new window: %+v", o.g) - } -} - -// Without the lease's word (a controller carried by another gate), the window counts from the applied report, -// and from the send at the least. -func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - r := gateRuleOf("mesh-controller", catalogue.Manifest{}) - if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) { - t.Errorf("floor %s after the send; want 1m", f.Sub(sent)) - } - if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"}, - map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) { - t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent)) - } -} - -// A controller that a witness put back, or that never becomes ready, fails its gate as before. -func TestAControllerThatFailsIsStillFailed(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - o := aGateOn("mesh-controller", "novox", sent) - o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back") - if o.g.Verdict != inventory.GateFailed { - t.Fatalf("a controller put back by its witness: %+v; want failed", o.g) - } - o = aGateOn("mesh-controller", "novox", sent) - o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready") - if o.g.Verdict != "" { - t.Fatalf("failed before the bound: %+v", o.g) - } - o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready") - if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") { - t.Fatalf("never ready: %+v; want failed at the bound", o.g) - } -} - -// threeReadings judges a module under the three-reading rule on one machine, the send at sent. -func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) { - t.Helper() - g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent} - rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) - floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) - return g, func(after time.Duration, h health) { - why := "" - if h != healthGood { - why = "the node-engine reports another build" - } - tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why, - rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after)) - } -} - -// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two -// minutes (novox/hq ADR 0352). -func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) { - withTheGatesTimes(t) - for _, module := range []string{"mesh-host", "dnsmasq"} { - g, judge := threeReadings(t, module, time.Now()) - judge(15*time.Second, healthGood) - judge(55*time.Second, healthGood) - if g.Verdict != "" || g.Passes != 2 { - t.Fatalf("%s: after two readings %+v; want still judging", module, g) - } - judge(95*time.Second, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 || - !strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") { - t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g) - } - } -} - -// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at -// two minutes comes sooner than the spacing after the last pass. -func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - last := sent.Add(100 * time.Second) - held := &inventory.PlanGate{Since: &sent, LastPass: &last} - if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) { - t.Fatal("the judging at two minutes is not due, or is due before") - } - held.Faulted = true - if dueAtTheMost(held, sent.Add(2*time.Minute)) { - t.Fatal("a send that was not healthy at a reading is judged at two minutes") - } - - g, judge := threeReadings(t, "mesh-host", time.Now()) - judge(100*time.Second, healthGood) - judge(2*time.Minute, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 2 { - t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g) - } - g, judge = threeReadings(t, "mesh-host", time.Now()) - judge(2*time.Minute, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 1 { - t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g) - } -} - -// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold. -func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) { - withTheGatesTimes(t) - g, judge := threeReadings(t, "mesh-host", time.Now()) - judge(10*time.Second, healthNotYet) - judge(100*time.Second, healthGood) - judge(2*time.Minute, healthGood) - if g.Verdict != inventory.GatePassed || g.Faulted { - t.Fatalf("%+v; want passed at two minutes", g) - } -} - -// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one -// never healthy fails at the bound and is put back; one found broken fails. -func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) { - withTheGatesTimes(t) - g, judge := threeReadings(t, "mesh-host", time.Now()) - judge(20*time.Second, healthNotYet) - judge(60*time.Second, healthGood) - judge(100*time.Second, healthGood) - judge(2*time.Minute+10*time.Second, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 3 { - t.Fatalf("%+v; want passed only on its third healthy reading", g) - } - g, judge = threeReadings(t, "mesh-host", time.Now()) - judge(60*time.Second, healthGood) - judge(100*time.Second, healthNotYet) - judge(2*time.Minute, healthGood) - if g.Verdict != "" || !g.Faulted { - t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g) - } - judge(10*time.Minute+time.Second, healthNotYet) - if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") { - t.Fatalf("never healthy again: %+v; want failed at the bound", g) - } - g, judge = threeReadings(t, "mesh-host", time.Now()) - judge(60*time.Second, healthGood) - judge(100*time.Second, healthBroken) - if g.Verdict != inventory.GateFailed { - t.Fatalf("a broken reading: %+v; want failed, to be put back", g) - } -} - -// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. -func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - o := aGateOn("node-tools", "ace", sent) - o.g.Applied["ace"] = sent.Add(10 * time.Second) - o.read(sent.Add(30*time.Second), healthGood, "") - if o.g.Verdict != "" { - t.Fatalf("the tool runner passed inside its witness window: %+v", o.g) - } - o.read(sent.Add(70*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") { - t.Fatalf("the tool runner: %+v; want passed on one reading", o.g) - } - - o = aGateOn(catalogue.DeliverySeat, "novox", sent) - o.read(sent.Add(5*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") { - t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g) - } -} - -// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at -// two minutes (novox/hq ADR 0352 decision 3). -func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} - modules := []string{"mesh-controller", "mesh-host"} - rules, floors := map[string]gateRule{}, map[string]time.Time{} - for _, m := range modules { - rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) - } - floors["mesh-host"] = sent - floors["mesh-controller"] = sent.Add(90 * time.Second) - judge := func(after time.Duration, h health) { - tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h}, - worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after)) - } - judge(30*time.Second, healthGood) - if g.Passes != 0 { - t.Fatalf("a reading before the controller's floor was counted: %+v", g) - } - judge(70*time.Second, healthNotYet) - judge(2*time.Minute, healthGood) - if !g.Faulted || g.Verdict != "" { - t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g) - } -} - -// A send carrying the controller and the node-engine is judged by the node-engine's three readings. -func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} - modules := []string{"mesh-controller", "mesh-host"} - rules, floors := map[string]gateRule{}, map[string]time.Time{} - for _, m := range modules { - rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) - floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) - } - for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} { - tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, - "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) - } - if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 { - t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g) - } -} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go deleted file mode 100644 index 8109ede3..00000000 --- a/cmd/mesh-controller/gate_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8). - -// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered, -// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is -// sent and reports it — with the gate's bounds shortened to judge in three steps. -type gateMesh struct { - open *stores - sent [][]string - health map[string]health // per machine, what a judging finds; healthy when unsaid - keeper *conditions.Keeper - told *conditions.Told -} - -func (g *gateMesh) plan(t *testing.T) inventory.Plan { - t.Helper() - p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate") - if err != nil { - t.Fatal(err) - } - return p -} - -// A module's build that fails its gate on the first machine is put back there — the previous build -// registered and sent to it again — and never reaches the second machine; it is marked, said as a -// condition and an event, never registered or rolled back again. - -// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then -// the rest are sent, the verdict kept, and the plan done. - -// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health -// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent. - -// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the -// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). - -// A first machine that never becomes healthy fails its gate at the bound, and is put back. - -// The health a judging finds, per core component and for a module. -func TestTheHealthDefinitions(t *testing.T) { - now := time.Now() - since := now.Add(-time.Minute) - applied := func(node string) gateFacts { - at := now - return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node, - Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, - served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}} - } - m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}} - - f := applied("anchor") - f.served["anchor"] = served{runtime: true, tools: map[string]bool{}} - if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") { - t.Errorf("a module whose tools are not served is %v (%s)", h, why) - } - f.served["anchor"].tools["app"] = true - if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { - t.Errorf("a module applied with its tools served is %v (%s)", h, why) - } - // A condition raised about it on that machine since it was sent: not yet healthy. - f.judged = true - f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ - Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}} - if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet { - t.Errorf("a module with a new condition about it is %v", h) - } - f.open[0].Raised = since.Add(-time.Hour) - if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { - t.Errorf("a condition older than the send counted against it: %v", h) - } - // A witness that put it back: broken. - f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb", - Outcome: lease.OutcomeRolledBack, Why: "x", At: now}} - if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken { - t.Errorf("a component a witness put back is %v", h) - } - // The node tools answer, or not. - f = applied("anchor") - if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet { - t.Errorf("node tools not answering are %v", h) - } - f.served["anchor"] = served{runtime: true} - if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood { - t.Errorf("node tools answering are %v", h) - } - // The controller: the lease held since the send, by a controller that says it is ready. - f = applied("control") - f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}} - if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet { - t.Errorf("a lease held by a controller older than the build is %v", h) - } - f.holder.Taken = now - if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood { - t.Errorf("a new controller holding the lease and ready is %v", h) - } - f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"} - if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet || - !strings.Contains(why, "first run") { - t.Errorf("a controller not ready is %v (%s)", h, why) - } - // What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running - // resource of it on that machine is stated healthy since the send; starting and unhealthy wait. - f = applied("anchor") - stated := func(state, reason string, heard time.Time) { - f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard, - Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason}, - {Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}} - } - plain := catalogue.Manifest{Module: "app"} - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood { - t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why) - } - stated(link.StateStarting, "", now) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") { - t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why) - } - stated(link.StateUnhealthy, "restarting", now) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") { - t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why) - } - stated(link.StateHealthy, "", since.Add(-time.Minute)) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet { - t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why) - } - stated(link.StateHealthy, "", now) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood { - t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why) - } - f.healthErr = errors.New("the store is away") - if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet { - t.Errorf("health that cannot be read is never read as healthy: %v", h) - } - // And the condition it raises after the send holds it, as every condition about it does. - f.healthErr = nil - f.judged = true - f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{ - Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}} - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") { - t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why) - } - - // A machine that refused what it was sent: broken. - f = applied("anchor") - r := f.reports["anchor"] - r.Outcome = inventory.OutcomeRefused - f.reports["anchor"] = r - if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken { - t.Errorf("a refusal is %v", h) - } -} - -// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused, -// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits. - -// A merge that deletes a module's directory builds nothing for it: the module is forgotten where -// nothing holds it, and a plan whose build finds no manifest goes on instead of failing. - -func mustJSON(t *testing.T, v any) []byte { - t.Helper() - b, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - return b -} - -func nodeID(t *testing.T, open *stores, name string) string { - t.Helper() - n, err := open.inventory.NodeByName(context.Background(), name) - if err != nil { - t.Fatal(err) - } - return n.ID -} - -// What a machine's witness says in its reports is a condition while it says it, by the host's words: -// `core...`, urgent for a rollback, a warning when it could not judge — and -// gone with the first report that no longer carries it. - -// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle -// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still -// open after a healthy reading. -func judgeLeavesAsTheCore(t *testing.T) { - t.Helper() - wasPasses, wasSettles := gateLeafPasses, gateLeafSettles - gateLeafPasses, gateLeafSettles = gatePasses, true - t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles }) -} diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go deleted file mode 100644 index ce649ce0..00000000 --- a/cmd/mesh-controller/generation.go +++ /dev/null @@ -1,306 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The assignment generation a declaration was composed from, and the record of every send (novox/hq -// issue 234). -// -// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the -// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and -// cannot tell a later send that carries an older view of the assignments. So a declaration now carries -// the generation of the assignments it was composed from — a counter the store raises in the same -// transaction as every assignment change — and a machine refuses one older than it applied, unless it is -// a gate's put-back. And every send is written down with who sent it, from which generation and naming -// which modules: the controller logged no send then, and which process sent the stale declaration could -// not be read back from anything the mesh kept. - -// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from. -const kindOlderGeneration = "older-generation" - -// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. -const generationRefusalsSaid = advisoryQuiet - -// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and -// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the -// record of the send whether or not the machine is sent them. -// -// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its -// declaration is allotted here like any other and carries the generation as it stands — never older than -// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never -// needs to send one. -func (o order) stamp(d *sendable) { - d.Sequence, d.Epoch = o.sequence, o.epoch - d.composedFrom, d.actingEpoch = o.generation, o.acting - d.Generation = 0 - if o.readsGeneration && o.generation > 0 { - d.Generation = o.generation - } -} - -// sentRecord is what the record of a send keeps beside its digest. -type sentRecord struct { - sequence int64 - epoch uint64 - generation int64 - // toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine - // whose node-engine has not said it reads one. - toldGeneration int64 - sender string - modules []string - // plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none. - plans []byte -} - -// sentRecordOf is a composed send's record: its sender is the caller this process acts for. -func sentRecordOf(ctx context.Context, d sendable) sentRecord { - r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, - toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} - if len(d.plans) > 0 { - kept := map[string]keptPlan{} - for module, p := range d.plans { - kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]} - } - // A map of plain strings and slices: it marshals. - r.plans, _ = json.Marshal(kept) - } - return r -} - -// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in -// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans). -type keptPlan struct { - catalogue.Plan - RequiredBy string `json:"required-by,omitempty"` -} - -// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the -// caller the controller ran this command for, or the account at the shell. -func callerOf(ctx context.Context) string { - if c := link.CallerIn(ctx); c != "" { - return c - } - return link.Caller() -} - -// senderOf is a send's sender in words: who asked, and the process and build that composed it — the -// answer issue 234 could not find, because two controllers and a one-shot push were all sending then. -func senderOf(caller string) string { - host, _ := os.Hostname() - return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version) -} - -// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender -// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that -// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the -// mesh cannot attribute is the louder fact, not a quieter one. -// -// **And raises the mesh's counter past what the machine applied, when the refused send was composed from -// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind -// the machine, which is a store put back from a backup — and every send after would be refused for ever. A -// stale sender's generation is below the counter, and the counter is left alone for it. -// -// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that -// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185. -// S20 names `push ` for that case instead. -func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { - r := report.OlderGeneration - if r == nil || inv == nil || report.Node == "" { - return - } - node, err := inv.NodeByName(ctx, report.Node) - if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+ - "read, so it is not raised: %v\n", report.Node, err) - return - } - var raised int64 - if now, err := inv.AssignmentGeneration(ctx); err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+ - "read: %v\n", report.Node, err) - } else if counterBehind(*r, now) { - if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+ - "and could not be raised: %v\n", now, report.Node, r.Applied, err) - raised = 0 - } else { - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+ - "a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n", - now, report.Node, r.Applied, raised, report.Node) - } - } - send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised) - if err == nil && !found { - send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence, - Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied, - CounterRaisedTo: raised}) - } - if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ - "kept, so it is not raised: %v\n", report.Node, report.Sequence, err) - return - } - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ - "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) -} - -// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the -// refused send carried the counter as it stands now, and the machine applied more than that. -func counterBehind(r link.GenerationRefusal, now int64) bool { - return r.Generation >= now && r.Applied > r.Generation -} - -// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was -// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal. -func watchGenerationRefusals(f *signalFacts) []conditions.Observation { - seen := map[string]bool{} - var out []conditions.Observation - for _, s := range f.refusedSends { - if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] { - continue - } - seen[s.NodeName] = true - o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, - Machine: s.NodeName, Severity: conditions.Warning, - Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ - "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ - "past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, - modulesWords(s.Modules)), - Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)), - Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), - Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ - "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), - Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)} - if s.CounterRaisedTo > 0 { - // Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and - // was raised; nothing has sent the machine its declaration since, so a person is asked to. - o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+ - "the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+ - "put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now", - s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo, - s.NodeName) - o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+ - "update and kept what it had. The record is repaired; a push sends the update again.", - s.NodeName, s.NodeName, s.NodeName) - } - out = append(out, o) - } - return out -} - -// modulesWords is a send's modules as a sentence says them. -func modulesWords(modules []string) string { - if len(modules) == 0 { - return "no module the mesh recorded" - } - return strings.Join(modules, ", ") -} - -// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and -// what it answered (issue 485): nothing when the mesh has not recorded a send to it. -func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { - s, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - return err - } - fmt.Fprintf(w, "%s:\n", node) - for _, line := range lastSendLines(s) { - fmt.Fprintln(w, line) - } - return nil -} - -// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when, -// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the -// machine answered of it — the refusal in its own words. -func lastSendLines(s inventory.Send) []string { - const stamp = "2006-01-02 15:04:05" - epoch := "under no lease epoch" - if s.Epoch > 0 { - epoch = fmt.Sprintf("under lease epoch %d", s.Epoch) - } - generation := "no assignment generation recorded" - if s.Generation > 0 { - generation = fmt.Sprintf("assignment generation %d", s.Generation) - } - lines := []string{ - fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)), - fmt.Sprintf(" by %s", s.Sender), - fmt.Sprintf(" %s, composed from %s", epoch, generation), - fmt.Sprintf(" naming %s", modulesWords(s.Modules)), - } - answer := func(words string) { lines = append(lines, " its answer "+words) } - switch a := s.Answer; { - case s.RefusedAt != nil: - // Before the report: a refusal for the generation is kept on the send itself and is the machine's - // answer to this send in particular. - answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one", - s.RefusedAt.Local().Format(stamp), s.RefusedApplied)) - case a.Outcome == "" || a.At == nil: - answer("the machine has not reported on it yet") - case a.Outcome == inventory.OutcomeApplied: - answer("applied it at " + a.At.Local().Format(stamp)) - case a.Outcome == inventory.OutcomeRefused: - answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:") - for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") { - lines = append(lines, " "+strings.TrimSpace(line)) - } - default: - answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome, - a.At.Local().Format(stamp), a.Failed)) - } - return lines -} - -// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line. -func sendAnswerWord(s inventory.Send) string { - switch a := s.Answer; { - case s.RefusedAt != nil: - return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied) - case a.Outcome == "" || a.At == nil: - return "not reported on yet" - case a.Outcome == inventory.OutcomeFailed: - return fmt.Sprintf("failed (%d resource(s))", a.Failed) - default: - return a.Outcome - } -} - -// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment -// generation and what it answered (novox/hq issue 485). `node show ` says the send in full. -func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) { - if unread != "" { - fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread) - return - } - if len(nodes) == 0 { - return - } - fmt.Println("last sent, per machine:") - for _, n := range nodes { - s, found := sends[n.Name] - if !found { - fmt.Printf(" %-12s no send recorded\n", n.Name) - continue - } - generation := "no generation" - if s.Generation > 0 { - generation = fmt.Sprintf("generation %d", s.Generation) - } - fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender, - generation, sendAnswerWord(s)) - } - fmt.Printf("\n `node show ` says a machine's last send in full\n\n") -} diff --git a/cmd/mesh-controller/generation_test.go b/cmd/mesh-controller/generation_test.go deleted file mode 100644 index a2372bcc..00000000 --- a/cmd/mesh-controller/generation_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The assignment generation a declaration was composed from (novox/hq issue 234). - -func bodyKeys(t *testing.T, s sendable) map[string]any { - t.Helper() - raw, err := s.Body() - if err != nil { - t.Fatal(err) - } - var keys map[string]any - if err := json.Unmarshal(raw, &keys); err != nil { - t.Fatal(err) - } - return keys -} - -// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older -// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate -// composes its put-back afresh, with the generation as it stands. -func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) { - resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}} - reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57} - var told sendable - told.Resources = resources - reads.stamp(&told) - keys := bodyKeys(t, told) - if keys["generation"] != float64(40) || keys["sequence"] != float64(12) { - t.Fatalf("a machine that reads a generation was sent %v", keys) - } - if _, there := keys["put_back"]; there { - t.Fatalf("a put-back mark was sent: %v", keys) - } - if told.composedFrom != 40 || told.actingEpoch != 57 { - t.Errorf("the send does not keep what it was composed from for its record: %+v", told) - } - - older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57} - var untold sendable - untold.Resources = resources - older.stamp(&untold) - if _, there := bodyKeys(t, untold)["generation"]; there { - t.Fatal("a machine whose node-engine never said it reads a generation was sent one") - } - if untold.composedFrom != 40 { - t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold) - } -} - -// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the -// process and build that composed it. -func TestASendNamesItsSender(t *testing.T) { - said := senderOf("g14.node-tools, through the mesh-controller seat") - if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") { - t.Fatalf("the sender reads %q", said) - } -} - -// refusedSend is a send a machine refused for its generation at a moment. -func refusedSend(at time.Time) inventory.Send { - return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40, - Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)", - Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true} -} - -// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation, -// against which the machine applied, and its sequence — the facts that took a morning to look for. -func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) { - now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) - f := calm(now) - f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))} - got := watchGenerationRefusals(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - o := got[0] - if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" { - t.Errorf("raised as %s about %q", o.Key(), o.Machine) - } - for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("the summary does not say %q: %s", want, o.Summary) - } - } - if o.Headline == "" || o.Explanation == "" || o.Resolved == "" { - t.Errorf("the condition is not worded for the operator: %+v", o) - } -} - -// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never -// only a line on stderr. -func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) { - now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) - f := calm(now) - s := refusedSend(now.Add(-time.Minute)) - s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil - f.refusedSends = []inventory.Send{s} - got := watchGenerationRefusals(f) - if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") { - t.Fatalf("an unattributed refusal was not raised as one: %+v", got) - } -} - -// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was -// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do. -func TestACounterBehindTheMachineAsksForAPush(t *testing.T) { - now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) - f := calm(now) - s := refusedSend(now.Add(-time.Minute)) - s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41 - f.refusedSends = []inventory.Send{s} - got := watchGenerationRefusals(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") || - strings.Contains(got[0].Explanation, "Nothing for you to do") { - t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation) - } -} - -// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below -// it, and the counter is left alone for that. -func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) { - for _, c := range []struct { - refused, applied, now int64 - behind bool - }{ - {refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40 - {refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past - {refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is - } { - r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied} - if got := counterBehind(r, c.now); got != c.behind { - t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind) - } - } -} diff --git a/cmd/mesh-controller/given.go b/cmd/mesh-controller/given.go index 88aec7c9..1c89e174 100644 --- a/cmd/mesh-controller/given.go +++ b/cmd/mesh-controller/given.go @@ -3,6 +3,8 @@ package main import ( "context" "encoding/json" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" "log" "strings" "sync" @@ -13,9 +15,8 @@ import ( // A value given by hand lives only until the module's first good start (novox/hq ADR 0228). // -// The serving controller hears every report; a clean one about the declaration a machine was last -// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is -// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`, +// The serving controller watches every have; a module's have that is running and healthy is the signal the store asks about +// (inventory.ReplaceGivenAfterStart). What it replaced is composed at once, said in the log and stated on the bus as the controller seat's `secret-replaced`, // so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log // is read as the count of repairs a healer is wanted for. @@ -41,33 +42,38 @@ var givenEvents link.Bus // same rows, and the store's claim makes one of them the replacer, but the sends need not race. var replacing sync.Map -// startedWell says a report is a machine's clean account of a declaration: everything applied, -// nothing failed or refused. Whether it is the declaration last sent is the store's to answer. -func startedWell(report link.Report) bool { - return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil +// startedOn is the signal of a module's first good start: its have is running and healthy, as of the time the node-engine +// wrote it. The zero time when it is not. +func startedOn(h sdkdelivery.Have, written time.Time) time.Time { + if h.Phase == sdkdelivery.PhaseRunning && h.Health == sdkdelivery.HealthOK { + return written + } + return time.Time{} } const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)" -// replaceGiven replaces what the report makes due, sends the machines, and says so. -func replaceGiven(ctx context.Context, open *stores, report link.Report) { - if _, busy := replacing.LoadOrStore(report.Node, true); busy { +// replaceGiven replaces what a module's good start makes due, composes the machines, and says so. +func replaceGiven(ctx context.Context, open *stores, node string, started map[string]time.Time) { + if _, busy := replacing.LoadOrStore(node, true); busy { return } - defer replacing.Delete(report.Node) - replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared) + defer replacing.Delete(node) + replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, node, started) if err != nil { - log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err) + log.Printf("a value given by hand on %s could not be replaced after its module started: %v", node, err) } for _, r := range replaced { - said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(), + said := SecretReplaced{Node: node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(), Sent: r.Machines, Why: givenWhy} - log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; sending %s", - r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", ")) - if err := sendTo(ctx, open, r.Machines); err != nil { + log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; composing %s", + r.Name, r.Module, node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", ")) + // Composed (novox/hq ADR 0539 clause 5): the compositions that use it get a new revision, which each node's + // want then names. + if _, err := composeModules(ctx, open, r.Machines); err != nil { said.Sent, said.Unsent = nil, err.Error() - log.Printf("the new %q of %s is sealed and not yet delivered to %s — the next push carries it: %v", - r.Name, r.Module, strings.Join(r.Machines, ", "), err) + log.Printf("the new %q of %s is sealed and its compositions on %s are not yet written — the next act that "+ + "composes them carries it: %v", r.Name, r.Module, strings.Join(r.Machines, ", "), err) } stateReplaced(ctx, said) } @@ -88,3 +94,60 @@ func stateReplaced(ctx context.Context, said SecretReplaced) { log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err) } } + +// watchHavesForGiven replaces the given values of a module on a node each time its have is running and healthy, until ctx +// ends: never on the watch's own goroutine. +func watchHavesForGiven(ctx context.Context, open *stores) { + bus, err := aBus() + if err != nil { + log.Printf("given values: the haves are not watched: %v", err) + return + } + js, err := jetstream.New(bus.Conn()) + if err != nil { + bus.Close() + return + } + kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket) + if err != nil { + bus.Close() + log.Printf("given values: the haves are not watched: %v", err) + return + } + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + bus.Close() + log.Printf("given values: the haves are not watched: %v", err) + return + } + go func() { + defer bus.Close() + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + node, module, isNode, err := sdkdelivery.ParseHaveKey(e.Key()) + if err != nil || isNode { + continue + } + have, err := sdkdelivery.DecodeHave(e.Value()) + if err != nil { + continue + } + at := startedOn(have, e.Created()) + if at.IsZero() { + continue + } + inBackground(func() { replaceGiven(ctx, open, node, map[string]time.Time{module: at}) }) + } + } + }() +} diff --git a/cmd/mesh-controller/given_test.go b/cmd/mesh-controller/given_test.go index 8a45a465..b1e664a8 100644 --- a/cmd/mesh-controller/given_test.go +++ b/cmd/mesh-controller/given_test.go @@ -1,10 +1,11 @@ package main import ( + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "strings" "testing" - - "github.com/novox/mesh-controller/internal/link" ) // A rotation asked through the seat carries why to the command, which records it in the hand-act @@ -27,22 +28,24 @@ func TestARotationThroughTheSeatCarriesWhy(t *testing.T) { } } -// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that -// the machine is there is not (novox/hq ADR 0228). -func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) { +// Only a have that is running and healthy is a good start, at the time it was written; any other is none (novox/hq ADR 0228, +// ADR 0539 §9). +func TestAGoodStartIsAHaveRunningAndHealthy(t *testing.T) { + at := time.Date(2026, 10, 24, 17, 0, 0, 0, time.UTC) for _, c := range []struct { - report link.Report - good bool + have sdkdelivery.Have + good bool }{ - {link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true}, - {link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true}, - {link.Report{Node: "anchor"}, false}, - {link.Report{Node: "anchor", Applied: []string{"x"}}, false}, - {link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false}, - {link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false}, + {sdkdelivery.Have{Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}, true}, + {sdkdelivery.Have{Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthFailed}, false}, + {sdkdelivery.Have{Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthWaits}, false}, + {sdkdelivery.Have{Phase: sdkdelivery.PhaseApplying, Health: sdkdelivery.HealthOK}, false}, + {sdkdelivery.Have{Phase: sdkdelivery.PhaseFailed, Health: sdkdelivery.HealthFailed}, false}, + {sdkdelivery.Have{Phase: sdkdelivery.PhaseHeld, Health: sdkdelivery.HealthOK}, false}, } { - if got := startedWell(c.report); got != c.good { - t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good) + got := startedOn(c.have, at) + if good := !got.IsZero(); good != c.good || good && !got.Equal(at) { + t.Errorf("%+v: started at %v, want a good start = %v", c.have, got, c.good) } } } diff --git a/cmd/mesh-controller/grant_follows_binding_integration_test.go b/cmd/mesh-controller/grant_follows_binding_integration_test.go index 0aed7741..f8441f2a 100644 --- a/cmd/mesh-controller/grant_follows_binding_integration_test.go +++ b/cmd/mesh-controller/grant_follows_binding_integration_test.go @@ -105,11 +105,6 @@ func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) { if got := declared.Received["store"]["postgres-database"]; len(got) != 0 { t.Fatalf("the anchor's store is still told about %+v", got) } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") || - !strings.Contains(said, "cleanup delete") { - t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said) - } // The store it is bound to: granted. grants, _, unbound, err = grantsOf(ctx, open, "laptop") diff --git a/cmd/mesh-controller/grants.go b/cmd/mesh-controller/grants.go index b81ddd6b..802bc549 100644 --- a/cmd/mesh-controller/grants.go +++ b/cmd/mesh-controller/grants.go @@ -126,28 +126,25 @@ func withVersion(g sdkdelivery.ControllerGrant) sdkdelivery.ControllerGrant { return g } -// grantsIssued writes the grants the bus's user list just sent to its machine issued (ADR 0528 clause 2). Said, never -// fatal: a grant not written now is written at the next send of the list, and the assigner's wants wait for it. -func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream, indent string) { +// grantsIssued writes the grants issued for every module whose credentials all exist at the fingerprint it is +// registered at (ADR 0528 clause 2, ADR 0539 clause 7). Its error is the caller's: with no send left, nothing writes a +// missed grant later but the caller's retry (the done key's take-in, composed_keys.go), and the assigner's wants wait. +func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream, indent string) error { kv, err := js.KeyValue(ctx, broker.GrantsBucket) if err != nil { - fmt.Printf("%sthe grants issued are not written: the bucket %s: %v\n", indent, broker.GrantsBucket, err) - return + return fmt.Errorf("the grants issued are not written: the bucket %s: %w", broker.GrantsBucket, err) } records, err := inv.BusRecords(ctx) if err != nil { - fmt.Printf("%sthe grants issued are not written: %v\n", indent, err) - return + return fmt.Errorf("the grants issued are not written: %w", err) } principals, err := broker.Users(records) if err != nil { - fmt.Printf("%sthe grants issued are not written: %v\n", indent, err) - return + return fmt.Errorf("the grants issued are not written: %w", err) } kept, err := inv.BusUsers(ctx) if err != nil { - fmt.Printf("%sthe grants issued are not written: %v\n", indent, err) - return + return fmt.Errorf("the grants issued are not written: %w", err) } hashes := make(map[string]string, len(kept)) for name, u := range kept { @@ -155,14 +152,14 @@ func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.Je } current, err := currentFingerprints(ctx, inv) if err != nil { - fmt.Printf("%sthe grants issued are not written: %v\n", indent, err) - return + return fmt.Errorf("the grants issued are not written: %w", err) } wrote, err := keepGrants(ctx, kv, grantsFrom(principals, hashes, current)) if wrote > 0 { fmt.Printf("%s%d grant(s) issued written for the assigner\n", indent, wrote) } if err != nil { - fmt.Printf("%ssome grants issued are not written: %v\n", indent, err) + return fmt.Errorf("some grants issued are not written: %w", err) } + return nil } diff --git a/cmd/mesh-controller/grants_integration_test.go b/cmd/mesh-controller/grants_integration_test.go deleted file mode 100644 index 3180a764..00000000 --- a/cmd/mesh-controller/grants_integration_test.go +++ /dev/null @@ -1,91 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1394/AC1 of C9 (ADR 0528 clause 2): when the controller has issued mesh-x's credential for fingerprint F (mesh-x -// registered at a build of F, its account minted, the bus's machine sent its declaration with the user list, through -// overTheBus.declare as a push sends it), then grants.mesh-x.F exists on the bus, -// and a second pass writes it again as nothing new. -func TestTheGrantOfAnIssuedCredentialIsOnTheBus(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - fp := grantFP('d') - m := catalogue.Manifest{Module: "mesh-x", Version: "1", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh-x/broker"}}} - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-x1", Repository: "novox/mesh-x", Module: "mesh-x", Commit: "c0ffee1", - SourceFingerprint: fp}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-x", BuiltFrom: "c0ffee1", Head: "c0ffee1"}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "mesh-x"); err != nil { - t.Fatal(err) - } - user := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: "mesh-x"}.Username() - if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, Node: "anchor", Module: "mesh-x"}); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - if err := broker.OnConn(conn).EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - // The live path: a send of the bus's machine's declaration carrying the user list (overTheBus.declare), which records - // the list sent and then writes the grants it issued. - legacy, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - _ = js.DeleteStream(ctx, "GRANTS_DECLARED") - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "GRANTS_DECLARED", Subjects: []string{link.DeclareSubject("anchor")}}); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "GRANTS_DECLARED") }) - send := overTheBus{open: open, server: link.SendingOn(link.OverNATS{Conn: conn, JS: legacy}), signer: aSignerOfZeros{}} - ready := readyNode{node: "anchor", declared: sendable{BusUsers: "the user list the bus's machine is sent"}} - if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil { - t.Fatalf("the send of the bus's machine: %v", err) - } - kv, err := js.KeyValue(ctx, broker.GrantsBucket) - if err != nil { - t.Fatal(err) - } - key, _ := sdkdelivery.ControllerGrantKey("mesh-x", fp) - e, err := kv.Get(ctx, key) - if err != nil { - t.Fatalf("no grant %s after the list was issued: %v", key, err) - } - g, err := sdkdelivery.DecodeControllerGrant(e.Value()) - if err != nil || len(g.Accounts) != 1 || g.Accounts[0] != user { - t.Fatalf("the grant reads %+v, %v; want the account %s", g, err, user) - } - if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil { - t.Fatalf("the second send: %v", err) - } - if again, _ := kv.Get(ctx, key); again.Revision() != e.Revision() { - t.Errorf("a second pass wrote the grant again (revision %d, was %d)", again.Revision(), e.Revision()) - } -} diff --git a/cmd/mesh-controller/grants_step_integration_test.go b/cmd/mesh-controller/grants_step_integration_test.go deleted file mode 100644 index 7e54c6d8..00000000 --- a/cmd/mesh-controller/grants_step_integration_test.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// The store test's manifests are manifests the controller takes in: checked here, where no store is needed, -// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2). -func TestTheGrantsStepFixturesAreManifests(t *testing.T) { - for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false), - "restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} { - m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1" - raw, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - if _, err := catalogue.ParseManifest(raw); err != nil { - t.Errorf("%s: %v", name, err) - } - } -} diff --git a/cmd/mesh-controller/grants_step_store_test.go b/cmd/mesh-controller/grants_step_store_test.go deleted file mode 100644 index 8432483d..00000000 --- a/cmd/mesh-controller/grants_step_store_test.go +++ /dev/null @@ -1,196 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/identity" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list, -// and nothing else — not the new build of a module it runs, not a module newly assigned there. -// -// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a -// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants -// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor -// makes the step fail, said, rather than install it unjudged. -func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult { - manifest["module"], manifest["version"] = module, "1" - raw, _ := json.Marshal(manifest) - return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module, - On: "anchor", Module: module, Commit: commit, Manifest: raw, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - } - for _, b := range []link.BuildResult{ - build("nats", "n1", start, natsFixture()), - build("restic", "c1", start.Add(time.Second), resticFixture(false)), - build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil { - t.Fatal(err) - } - } - - wasEpoch := epochForActs - epochForActs = func(context.Context) (uint64, error) { return 7, nil } - t.Cleanup(func() { epochForActs = wasEpoch }) - - // A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped. - compose := func(under context.Context) (catalogue.Resolution, sendable) { - t.Helper() - numbered, err := allot(under, inv, "anchor") - if err != nil { - t.Fatal(err) - } - plan, _, err := resolveNode(under, open, "anchor") - if err != nil { - t.Fatal(err) - } - declared, err := composeSend(under, open, "anchor") - if err != nil { - t.Fatal(err) - } - numbered.stamp(&declared) - return plan, declared - } - record := func(declared sendable) { - t.Helper() - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch, - sentRecordOf(ctx, declared)); err != nil { - t.Fatal(err) - } - } - // Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop - // too (who is on the private network), and a step that kept every machine's builds refused anchor's - // composition for want of laptop's last send (repo-check of #230 at 0f853e93). - kept := keepingEveryBuild(keepingRecorded(ctx), "anchor") - - // What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it. - _, before := compose(keepingRecorded(ctx)) - record(before) - plan, declared := compose(kept) - if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" { - t.Fatalf("with nothing changed, the step reads %q, %v", only, err) - } - - // The merge: restic's c2 gives it a health tool. - if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute), - resticFixture(true))); err != nil { - t.Fatal(err) - } - - // The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool. - generation, err := inv.AssignmentGeneration(ctx) - if err != nil { - t.Fatal(err) - } - plan, declared = compose(kept) - for _, m := range plan.Modules { - if m.Module == "restic" && len(m.Tools) > 0 { - t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools) - } - } - if declared.Builds["restic"] != "c1" { - t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"]) - } - if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") { - t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers) - } - // The generation it carries is the current one (novox/hq issue 234), and the step does not raise it. - if declared.composedFrom != generation { - t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation) - } - // The guard, on the very declaration the send sends: only the user list differs from the last send. - if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" { - t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err) - } - // And the gate's refusal still reads the step's send: it moves nothing there. - if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" { - t.Fatalf("the step's send is refused as a move: %v, %v", names, err) - } - // Recorded as the send records it: anchor still runs restic c1, its gate still to come there. - record(declared) - if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" { - t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err) - } - if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation { - t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err) - } - - // A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent. - if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil { - t.Fatal(err) - } - plan, declared = compose(kept) - only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") { - t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only) - } - // And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled. - identity.ForTest(t) - if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) || - !strings.Contains(err.Error(), "wallpaper") { - t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err) - } -} - -// fixtureImage is the image every fixture container of the grants step's tests runs. -var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64) - -// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by -// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone -// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither. -func resticFixture(withTool bool) map[string]any { - backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage} - measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage} - m := map[string]any{"resources": []any{backup, measure}} - if withTool { - backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"} - measure["health"] = map[string]any{"kind": "runtime"} - m["tools"] = []string{"backup_health"} - } - return m -} - -// natsFixture is the bus's module: it holds mesh-broker and is sent the user list. -func natsFixture() map[string]any { - return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf", - "claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}} -} - -// wallpaperFixture is a module the bus's machine is newly assigned. -func wallpaperFixture() map[string]any { - return map[string]any{"resources": []any{ - map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}} -} diff --git a/cmd/mesh-controller/grants_step_test.go b/cmd/mesh-controller/grants_step_test.go deleted file mode 100644 index c8502758..00000000 --- a/cmd/mesh-controller/grants_step_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The grants reach the bus before the gate (novox/hq issue 490). -// -// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to -// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused -// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and -// that machine runs restic too, so it was left out of the send while restic's new build waited there for -// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged. - -// aSend is one send the walk made: to which machines, and whether every build there was kept. -type aSend struct { - names []string - keepsAll bool - judged []string -} - -// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine -// holding the bus is novox, and its list is behind as behind says. -func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend { - t.Helper() - var sends []aSend - wasSend, wasBehind := sendRollout, brokerBehindOf - t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind }) - brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) { - if slices.Contains(names, holder) { - return holder, false, nil - } - return holder, behind, nil - } - sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) { - s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])} - for n := range scopeOf(ctx).judged { - s.judged = append(s.judged, n) - } - sends = append(sends, s) - if refuse != nil && s.keepsAll { - return nil, refuse - } - return names, nil - } - return &sends -} - -// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox. -func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) { - sends := sendsRecorded(t, "novox", true, nil) - sent, grants, err := sendJudged(t.Context(), nil, "ace") - if err != nil { - t.Fatal(err) - } - if len(*sends) != 2 { - t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends) - } - first, then := (*sends)[0], (*sends)[1] - if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 { - t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first) - } - if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" { - t.Errorf("the second send is %+v, want ace, judged", then) - } - if strings.Join(sent, ",") != "ace" { - t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent) - } - if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" { - t.Fatalf("the gate keeps %+v as its grants step", grants) - } - line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}) - if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") { - t.Errorf("plans says the gate as %q", line) - } -} - -// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus. -func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) { - for _, c := range []struct { - name string - node string - behind bool - }{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} { - t.Run(c.name, func(t *testing.T) { - sends := sendsRecorded(t, "novox", c.behind, nil) - _, grants, err := sendJudged(t.Context(), nil, c.node) - if err != nil { - t.Fatal(err) - } - if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil { - t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants) - } - }) - } -} - -// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before. -func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) { - sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person")) - sent, grants, err := sendJudged(t.Context(), nil, "ace") - if err != nil { - t.Fatal(err) - } - if strings.Join(sent, ",") != "ace" || len(*sends) != 2 { - t.Errorf("the walk made %+v", *sends) - } - if grants == nil || grants.Failed == "" || grants.At != nil { - t.Fatalf("the gate keeps %+v", grants) - } - if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line, - "grants step to novox FAILED, sent without it: a bus upgrade waits for a person") { - t.Errorf("plans says the gate as %q", line) - } -} - -// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the -// gate says it FAILED with what differed, and the walk goes on. -func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) { - refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly) - sends := sendsRecorded(t, "novox", true, refusal) - sent, grants, err := sendJudged(t.Context(), nil, "ace") - if err != nil { - t.Fatal(err) - } - if strings.Join(sent, ",") != "ace" || len(*sends) != 2 { - t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends) - } - if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") { - t.Fatalf("the gate keeps %+v", grants) - } -} - -// The step's exemption is the list's content alone: its path or mode moving is a change like any other. -func TestOnlyTheUserListsContentIsExempt(t *testing.T) { - list := "nats.bus-users" - file := func(fields map[string]any) sentResource { - r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"} - for k, v := range fields { - r[k] = v - } - body, _ := json.Marshal(map[string]any{"resources": []any{r}}) - s, err := summarize(body) - if err != nil { - t.Fatal(err) - } - return s[0] - } - was := file(nil) - if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 { - t.Errorf("the list's new content reads as more: %v", other) - } - for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} { - other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list) - if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) { - t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other) - } - } -} - -// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed -// as any send composes them. -func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) { - ctx := keepingEveryBuild(t.Context(), "novox") - if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") { - t.Error("the grants step keeps the builds of a machine it does not send") - } -} diff --git a/cmd/mesh-controller/group_one_batch_test.go b/cmd/mesh-controller/group_one_batch_test.go deleted file mode 100644 index b7c69e57..00000000 --- a/cmd/mesh-controller/group_one_batch_test.go +++ /dev/null @@ -1,139 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A group is one batch and one plan (novox/hq ADR 0418 clauses 3 and 6): the batch holding a merge of a group with a -// member still to merge does not close by the merge window, however long that member takes, and a group with a -// member of each kind is one batch whatever order its members are heard in. - -func TestAGroupsBatchWaitsForItsLastMemberHoweverLongItTakes(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - states := map[string]string{"novox/one": "ready", "novox/two": "ready"} - groupStates(t, "feat/g", states) - first := fromHead(repoMerge("one", "c1", t0), "feat/g") - hear(t, open, first, t0) - // Past the window and past its maximum, with the second member still to merge: one batch, no walk. - for _, after := range []time.Duration{90 * time.Second, 11 * time.Minute, 30 * time.Minute} { - cutAt(t, open, t0.Add(after)) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 1 { - t.Fatalf("%s after the first member: %d walk(s), %d batch(es) %+v", after, len(ws), len(bs), bs) - } - } - // The last member merges: the window runs again from that merge. - at := t0.Add(30 * time.Minute) - states["novox/two"] = "merged" - hear(t, open, fromHead(repoMerge("two", "c2", at), "feat/g"), at.Add(time.Second)) - cutAt(t, open, at.Add(time.Second+89*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window ran again: %+v", ws) - } - cutAt(t, open, at.Add(time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" { - t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } -} - -// A group with a member of each kind is one batch and one walk, in both orders of hearing, started without -// mesh-delivery's word. -func TestAGroupWithAMemberOfEachKindIsOneBatchAndOneWalk(t *testing.T) { - for name, ownFirst := range map[string]bool{"controller member first": true, "catalogue member first": false} { - t.Run(strings.ReplaceAll(name, " ", "_"), func(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} - groupStates(t, "feat/both-kinds", states) - own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds") - cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds") - first, second := cat, own - if ownFirst { - first, second = own, cat - } - second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano) - hear(t, open, first, t0) - states["novox/"+second.Repo] = "merged" - hear(t, open, second, t0.Add(10*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("both members heard: %d walk(s), %d batch(es) %+v; want one batch of both", len(ws), len(bs), bs) - } - cutAt(t, open, t0.Add(10*time.Second+90*time.Second)) - ws, bs = walks(t, open) - if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/mesh-controller") != "k1" || - ws[0].CommitOf("novox/mesh-catalog") != cat.Commit { - t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } - if ws[0].Delivery != nil && ws[0].Delivery.Awaits != "" { - t.Fatalf("a plan carrying the controller waits for %s's word", ws[0].Delivery.Awaits) - } - }) - } -} - -// A group whose last member has not merged ten minutes after its first raises a condition naming it, and the held -// batch is not the batch S18 reads as a cut the controller failed to make. -func TestAHeldBatchRaisesTheGroupConditionAndNotTheBatchNotCutOne(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) - now := t0.Add(12 * time.Minute) - batches, err := gatherBatches(t.Context(), open.inventory, now) - if err != nil || len(batches) != 1 { - t.Fatalf("batches %+v: %v", batches, err) - } - f := &signalFacts{now: now, batches: batches} - if got := watchBatchesNotCut(f); len(got) != 0 { - t.Fatalf("a held batch is read as not cut: %+v", got) - } - got := watchGroupsNotMerged(f) - if len(got) != 1 || got[0].Kind != kindGroupNotMerged || !strings.Contains(got[0].Summary, "novox/two") || - !strings.Contains(got[0].Summary, "feat/g") { - t.Fatalf("the group's condition: %+v", got) - } - if early := watchGroupsNotMerged(&signalFacts{now: t0.Add(5 * time.Minute), batches: batches}); len(early) != 0 { - t.Fatalf("raised before the maximum: %+v", early) - } -} - -// The batch held for a group survives a restart: the hold is in the record. -func TestAHeldBatchSurvivesARestart(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) - reopened, err := open.inventory.Batches(t.Context()) - if err != nil || len(reopened) != 1 || len(reopened[0].Delivery.Batch.Holds) != 1 { - t.Fatalf("the record of the held batch: %+v %v", reopened, err) - } - cutAt(t, open, t0.Add(time.Hour)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 || bs[0].State != inventory.PlanAssembling { - t.Fatalf("after an hour: %d walk(s), batches %+v", len(ws), bs) - } -} - -// A mesh-delivery that does not yet say the group's gate is composed merges member two only after member one is -// delivered, so a batch held for it would wait for ever: the merge waits for the window, as before. -func TestAnOlderMeshDeliverysGroupIsNotHeldOpen(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - groupStatesFrom(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}, false) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) - if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 0 { - t.Fatalf("held for an older mesh-delivery's group: %+v", bs) - } - cutAt(t, open, t0.Add(90*time.Second)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("at its window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } -} diff --git a/cmd/mesh-controller/group_wait.go b/cmd/mesh-controller/group_wait.go deleted file mode 100644 index 8713318e..00000000 --- a/cmd/mesh-controller/group_wait.go +++ /dev/null @@ -1,228 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A delivery group still assembles** (novox/hq ADR 0306 decision 2): a merge whose delivery group has a member -// not yet merged waits for the merge window, whether or not a batch is assembling, so the group's members walk in -// one batch, in their order (ADR 0276 decision 5). Which groups exist and where each member stands is -// mesh-delivery's record (ADR 0239): a group is the pull requests of two repositories or more sharing a head -// branch name, which is its id. The controller asks it, through its `groups` verb, when a merge from a head is -// heard; a member whose pull request is open (proposed, checked, ready or rejected) is still to merge. When -// mesh-delivery does not answer within groupAskWithin, or answers something unreadable, the merge waits: the safe -// error is to wait, and its batch says why. - -// groupAskWithin is how long mesh-delivery is given to say a merge's group, while the merge is being heard. -var groupAskWithin = 3 * time.Second - -// openPullStates are the states of a delivery whose pull request is open and not merged (mesh-delivery's table): -// a member in one of them is still to merge. Closed unmerged is stopped, final, and waits for nothing. -var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready": true, "rejected": true} - -// groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `@ (state)`. -type groupMember struct { - Repository, State string - // Composed says the group's line says its gate is the composed check's (a mesh-delivery that merges a group in - // one pass): only such a group is held open for its last member, since an older mesh-delivery merges member two - // only after member one is delivered, and a held batch would wait for ever. - Composed bool -} - -// groupsLine is one group as mesh-delivery's `groups` says it: its id (the head branch name), its members as text and, -// from a mesh-delivery that knows novox/hq ADR 0382, each member again as an object (`member_details`) naming its pull -// request's number, whole head, base branch and whether it is open. -type groupsLine struct { - ID string `json:"id"` - Members []string `json:"members"` - Details []memberLine `json:"member_details,omitempty"` - // ComposedGate says the group's gate is the composed check's (novox/hq ADR 0403): a member's own check runs the - // repository's check alone. Said by a mesh-delivery that merges a group in one pass; absent from an older one, whose - // members keep their own gate. - ComposedGate bool `json:"composed_gate,omitempty"` -} - -// memberLine is one member of a group, as `groups` says it beside its text. -type memberLine struct { - ID string `json:"id"` - Repository string `json:"repository"` - State string `json:"state"` - Number int `json:"number"` - Head string `json:"head"` - Base string `json:"base"` - Open bool `json:"open"` -} - -// groupsAsked is mesh-delivery's `groups`: every group it keeps, or an error when it cannot be asked or did not -// answer within groupAskWithin. A seam a test replaces. -var groupsAsked = func(ctx context.Context, held bool) ([]groupsLine, error) { - if !held { - return nil, fmt.Errorf("no holder of %s is on record to say it", catalogue.DeliverySeat) - } - // On the bus as this process is: the serving controller's own connection, lent, or — in a verb that runs as a command - // of its own, such as a recheck's `delivery check` — one dialled for the moment (onTheBus). - var raw json.RawMessage - err := onTheBus(func(conn *nats.Conn) error { - asking, cancel := context.WithTimeout(ctx, groupAskWithin) - defer cancel() - var err error - raw, err = askDeliveryOwner(asking, conn, "groups", map[string]any{}) - return err - }) - if err != nil { - if errors.Is(err, link.ErrNothingServes) { - return nil, fmt.Errorf("nothing serves %s.groups", catalogue.DeliverySeat) - } - return nil, err - } - var lines []groupsLine - if err := json.Unmarshal(raw, &lines); err != nil { - return nil, fmt.Errorf("%s.groups answered something unreadable: %w", catalogue.DeliverySeat, err) - } - return lines, nil -} - -// deliveryGroupOf is what mesh-delivery says of the group a head names: its members, and whether the head names a -// group at all; an error when it cannot be asked or did not answer. A seam a test replaces. -var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]groupMember, bool, error) { - lines, err := groupsAsked(ctx, held) - if err != nil { - return nil, false, err - } - for _, l := range lines { - if l.ID != head { - continue - } - var members []groupMember - for _, s := range l.Members { - m := memberOf(s) - m.Composed = l.ComposedGate - members = append(members, m) - } - return members, true, nil - } - return nil, false, nil -} - -// memberOf reads one member as `groups` says it: `novox/mesh-sdk@d41d255f28ad (held)`. -func memberOf(s string) groupMember { - id, state := strings.TrimSpace(s), "" - if i := strings.LastIndex(id, " ("); i >= 0 && strings.HasSuffix(id, ")") { - id, state = id[:i], id[i+2:len(id)-1] - } - repository, _, _ := strings.Cut(id, "@") - return groupMember{Repository: repository, State: state} -} - -// groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing -// when it names no head, its head names no group, or every other member is merged or closed. -// -// A member of another repository whose merge this controller already keeps in a batch not yet cut, of either -// kind, is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins -// that batch by their head. **Kinds do not part a group** (novox/hq ADR 0418 clause 6): a controller member and a -// catalogue member share one batch. -// -// Asked before the plans are held: a slow answer holds up no walk. -func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) { - why, _, err := groupWaitHolding(ctx, inv, m, held) - return why, err -} - -// groupWaitHolding is groupWait, and whether the wait is for a member still to merge: such a batch is held open -// however long it takes (ADR 0418 clause 3). A wait only because mesh-delivery did not say is no hold. -func groupWaitHolding(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held bool) (string, bool, error) { - if m.Head == "" { - return "", false, nil - } - open, isGroup, composed, err := openMembersOf(ctx, inv, held, m.Head, m.Owner+"/"+m.Repo) - if err != nil { - return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+ - "window, the safe side", catalogue.DeliverySeat, m.Head, err), false, nil - } - if !isGroup || len(open) == 0 { - return "", false, nil - } - return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), composed, nil -} - -// openMembersOf are the members of the group a head names that are still to merge, in words: those whose pull -// request is open and whose merge no batch not yet cut keeps. self is a repository to leave out, the merge being heard. -func openMembersOf(ctx context.Context, inv *inventory.Inventory, held bool, head, self string) ([]string, bool, bool, error) { - kept, err := keptFromHead(ctx, inv, head) - if err != nil { - return nil, false, false, err - } - members, isGroup, err := deliveryGroupOf(ctx, held, head) - if err != nil || !isGroup { - return nil, isGroup, false, err - } - var open []string - composed := false - for _, g := range members { - composed = composed || g.Composed - if strings.EqualFold(g.Repository, self) || kept[strings.ToLower(g.Repository)] || !openPullStates[g.State] { - continue - } - open = append(open, g.Repository+" ("+g.State+")") - } - sort.Strings(open) - return open, true, composed, nil -} - -// keptFromHead are the repositories (lower case) of the merges from a head that batches not yet cut hold, of -// either kind. -func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string) (map[string]bool, error) { - batches, err := inv.Batches(ctx) - if err != nil { - return nil, err - } - kept := map[string]bool{} - for _, b := range batches { - merges, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return nil, err - } - for _, k := range merges { - if announcedOf(k).Head == head { - kept[strings.ToLower(k.Repository)] = true - } - } - } - return kept, nil -} - -// batchOfGroup is the batch not yet cut that holds a merge of another repository from the same head: the group's -// batch, whatever kind it is. -func batchOfGroup(ctx context.Context, inv *inventory.Inventory, head, repository string) (inventory.Plan, bool, error) { - if head == "" { - return inventory.Plan{}, false, nil - } - batches, err := inv.Batches(ctx) - if err != nil { - return inventory.Plan{}, false, err - } - for _, b := range batches { - merges, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return inventory.Plan{}, false, err - } - for _, k := range merges { - if announcedOf(k).Head == head && !strings.EqualFold(k.Repository, repository) { - return b, true, nil - } - } - } - return inventory.Plan{}, false, nil -} diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 97e34a25..6da4383c 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -55,23 +55,9 @@ const causeDrill = "drill" // repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or // listed without a decision, counts, so a new verb is a repair until its entry says otherwise. var handActVerbs = []handActVerb{ - // Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by - // default (ADR 0236) exists to end — except a push that only moved builds a `record` policy held for - // a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go). - {Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " + - "upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded}, - // Stopping or starting a walk the operator chose on a warrant (novox/hq ADR 0259) is their decision. - {Verb: "plans stop", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)", - DecidedFor: []string{conditions.CauseOperatorAnswer}}, - {Verb: "plans close"}, // A module registered at references by hand, because the builder had no key for it (ADR 0526 clause 3): a repair // the builder should have made, so it counts. {Verb: "register"}, - // A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or - // not trusted with it — either is a repair the owner should have made. Unless the operator chose it on - // a warrant (ADR 0259). - {Verb: "plans go", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)", - DecidedFor: []string{conditions.CauseOperatorAnswer}}, // An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a // channel that proved who answered, performed by the controller as itself. {Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"}, @@ -200,19 +186,6 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) { } act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why), Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)} - // A push naming one machine says whether it only moves recorded builds (novox/hq issue 301): - // read from what it carries, before it is sent. - if verb == "push" && len(args) == 1 && !strings.HasPrefix(args[0], "-") { - switch carried, why, err := recordedPushOf(ctx, args[0]); { - case err != nil: - fmt.Fprintf(os.Stderr, "whether this push only moves recorded builds could not be read, so it is "+ - "recorded as a push by hand: %v\n", err) - case len(carried) > 0: - act.Kind, act.Carried = link.KindRecordedBuilds, carried - default: - fmt.Printf("a push by hand, not of recorded builds only: %s\n", why) - } - } err := onTheBus(func(conn *nats.Conn) error { written, err := link.RecordHandAct(ctx, conn, act) act = written @@ -319,13 +292,6 @@ func listHandActs(ctx context.Context, args []string, w io.Writer) error { fmt.Fprintf(w, ", condition %s", a.Condition) } fmt.Fprintln(w, ")") - if pushedRecorded(a) { - carried := strings.Join(a.Carried, "; ") - if carried == "" { - carried = recordedBefore[a.ID] - } - fmt.Fprintf(w, " a push of recorded builds, no repair: %s\n", carried) - } } if len(repeated) > 0 { causes := make([]string, 0, len(repeated)) diff --git a/cmd/mesh-controller/handacts_test.go b/cmd/mesh-controller/handacts_test.go index 19d5a4f5..a4055ffe 100644 --- a/cmd/mesh-controller/handacts_test.go +++ b/cmd/mesh-controller/handacts_test.go @@ -4,49 +4,8 @@ import ( "context" "strings" "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" ) -// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before -// anything is done, through the seat, through `command`, and at a shell where nothing automated runs -// the verb. -func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - }{ - {"push", map[string]any{"node": "anchor"}}, - {"push", map[string]any{}}, - {"plans", map[string]any{"close": "plan-1"}}, - {"plans", map[string]any{"stop": "plan-1"}}, - {"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}}, - } { - argv, err := argvFor(c.verb, c.args) - if c.verb == "plans" && err == nil { - // The seat composes the command line; the command refuses it, before opening anything. - err = plansCommand(context.Background(), argv[1:]) - } - if err == nil || !strings.Contains(err.Error(), "why") { - t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err) - } - } - for _, args := range [][]string{{"EVENTS", "controller"}} { - if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") { - t.Errorf("consumer-reset without why: %v", err) - } - } - if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil || - !strings.Contains(err.Error(), "--why") { - t.Errorf("hand-act record without why: %v", err) - } - if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil || - !strings.Contains(err.Error(), "--cause") { - t.Errorf("hand-act record without a cause: %v", err) - } -} - // With a reason, the seat passes it to the command, and a verb that only reads is not held to one. func TestARepairByHandCarriesItsReason(t *testing.T) { for _, c := range []struct { @@ -54,11 +13,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) { args map[string]any want string }{ - {"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"}, - "push anchor --wait 0s --why stuck --cause sent-not-reported"}, - {"plans", map[string]any{"close": "plan-1", "why": "the report will not come"}, - "plans close plan-1 --why the report will not come"}, - {"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"}, {"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"}, "hand-act record restarted --why hung --cause proxy --condition machine.a.silent"}, {"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"}, @@ -70,20 +24,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) { } } -// The summary of durations says, per kind and subject, what a bound would be set from. -func TestDurationsAreSummarisedPerSubject(t *testing.T) { - var ds []inventory.Duration - for i := 1; i <= 10; i++ { - ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor", - Took: time.Duration(i) * time.Second}) - } - got := summarise(ds) - if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 || - got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" { - t.Fatalf("%+v", got) - } -} - // **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses // the cause, so a repair cannot pass for a drill by the word it gives. func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) { diff --git a/cmd/mesh-controller/handover_test.go b/cmd/mesh-controller/handover_test.go index 449f2f9e..053041ba 100644 --- a/cmd/mesh-controller/handover_test.go +++ b/cmd/mesh-controller/handover_test.go @@ -5,10 +5,7 @@ import ( "errors" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -69,28 +66,6 @@ func TestAHandOverIsRefusedThroughEveryVerb(t *testing.T) { } } -// The condition's words are plain and name no machine's binary; the operator's line, with the node and the path -// (novox/hq ADR 0272), is in the summary the module's health gives (moduleHealthWord) and in the evidence. -func TestTheUsedAsFoundConditionNamesTheOperatorsLine(t *testing.T) { - rs := []inventory.ResourceHealth{foundDirectory("notes", time.Now().Add(-24*time.Hour))} - o := usedAsFoundObservation("notes", "laptop", "notes on laptop uses notes.data as found", rs) - if strings.Contains(o.Needs, "mesh-host") || strings.Contains(o.Explanation, "mesh-host") || - !strings.Contains(o.Needs, "control-node") { - t.Fatalf("the condition's words: %q %q", o.Needs, o.Explanation) - } - if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, - Resolved: o.Resolved}, "laptop"); !ok { - t.Fatalf("not plain: %s", why) - } - f := gateFacts{now: time.Now(), health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), - Resources: rs}}} - h, why := moduleHealthWord("notes", "laptop", time.Now().Add(-time.Hour), f) - if h != healthPerson || !strings.Contains(why, "`nox node hand-over laptop ` on the control-node") || - strings.Contains(why, "mesh-host") || strings.Contains(why, "/srv/") { - t.Fatalf("the module's health reads %v %q; want the operator's line", h, why) - } -} - // **Nothing is asked of a node the mesh does not know, and the engine's refusal is the command's failure** // (review of issue 356): a refused hand-over never exits as a success. func TestAHandOverAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) { diff --git a/cmd/mesh-controller/haves.go b/cmd/mesh-controller/haves.go new file mode 100644 index 00000000..2ccbfe7e --- /dev/null +++ b/cmd/mesh-controller/haves.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "errors" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// What a machine has, against what it is wanted to have (novox/hq ADR 0539 §1, §9). Nothing is sent, so nothing records that a +// machine was sent anything: whether a machine has "acted on the current words" is read from its haves, the node-engine's +// own bucket, beside the assigner's wants. A node is current when every module wanted on it is had at the wanted fingerprint +// and composition, applied (not failed, not held, not still applying), and every module wanted removed is gone. A node that +// nothing is wanted on has nothing to apply and is current. + +// currentNodes is, for each node a want names, whether it has everything wanted of it. Pure. +func currentNodes(wants map[string]sdkdelivery.Want, haves map[string]sdkdelivery.Have) map[string]bool { + out := map[string]bool{} + for key, w := range wants { + node, module, err := sdkdelivery.ParseWantKey(key) + if err != nil { + continue // the node's own want, or a fingerprint the assigner holds: not a module's want + } + if _, seen := out[node]; !seen { + out[node] = true + } + haveKey, err := sdkdelivery.HaveKey(node, module) // `.`, where a want's key is `want..` + if err != nil { + continue + } + h, had := haves[haveKey] + switch { + case w.Removed: + if had && h.Phase != sdkdelivery.PhaseRemoved { + out[node] = false + } + case !had, h.FP != w.FP, h.Composed != w.Composed, h.Phase != sdkdelivery.PhaseRunning: + out[node] = false + } + } + return out +} + +// readWantsAndHaves reads the assigner's wants and the node-engines' haves from the bus; a bucket not there yet is empty. +func readWantsAndHaves(ctx context.Context, js jetstream.JetStream) (map[string]sdkdelivery.Want, map[string]sdkdelivery.Have, error) { + wants := map[string]sdkdelivery.Want{} + haves := map[string]sdkdelivery.Have{} + each := func(bucket string, take func(key string, value []byte)) error { + kv, err := js.KeyValue(ctx, bucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil + } + if err != nil { + return err + } + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil + } + if err != nil { + return err + } + for _, key := range keys { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + continue + } + if err != nil { + return err + } + take(key, e.Value()) + } + return nil + } + if err := each(sdkdelivery.AssignerWantsBucket, func(key string, value []byte) { + if w, err := sdkdelivery.DecodeWant(value); err == nil { + wants[key] = w + } + }); err != nil { + return nil, nil, err + } + if err := each(sdkdelivery.HaveBucket, func(key string, value []byte) { + if h, err := sdkdelivery.DecodeHave(value); err == nil { + haves[key] = h + } + }); err != nil { + return nil, nil, err + } + return wants, haves, nil +} + +// applyCurrent sets Current of each report from the nodes' states: a node some want names is current as currentNodes says; a +// node nothing is wanted on that has reported has nothing to apply and is current. Pure. +func applyCurrent(reports []inventory.Reported, current map[string]bool) { + for i := range reports { + c, wanted := current[reports[i].Node] + reports[i].Current = c || !wanted && reports[i].At != nil + reports[i].CurrentUnknown = false + } +} + +// currentUnknown marks every report's Current as not known: the haves could not be read. Pure. +func currentUnknown(reports []inventory.Reported) { + for i := range reports { + reports[i].Current, reports[i].CurrentUnknown = false, true + } +} + +// lastReports is inventory.LastReports with Current read from the haves: every machine's last report, and whether the machine +// has everything the assigner wants of it. A process that cannot reach the bus cannot tell, and says so (CurrentUnknown). +func lastReports(ctx context.Context, inv *inventory.Inventory) ([]inventory.Reported, error) { + reports, err := inv.LastReports(ctx) + if err != nil { + return nil, err + } + bus, err := aBus() + if err != nil { + currentUnknown(reports) + return reports, nil // empty-on-error: the bus is not reachable, so whether a machine is current is marked not known + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return nil, err + } + wants, haves, err := readWantsAndHaves(ctx, js) + if err != nil { + return nil, err + } + applyCurrent(reports, currentNodes(wants, haves)) + return reports, nil +} diff --git a/cmd/mesh-controller/haves_test.go b/cmd/mesh-controller/haves_test.go new file mode 100644 index 00000000..6a33006b --- /dev/null +++ b/cmd/mesh-controller/haves_test.go @@ -0,0 +1,181 @@ +package main + +import ( + "time" + + "errors" + "github.com/novox/mesh-controller/internal/inventory" + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **A machine is current when it has what is wanted of it** (novox/hq ADR 0539 §1, §9, issue 1945): every module wanted on it is +// had at the wanted fingerprint and composition, running, and every module wanted removed is gone. Read from the wants and the +// haves; there is no send to compare a report with. +func TestAMachineIsCurrentWhenItHasWhatIsWantedOfIt(t *testing.T) { + fp := func(c string) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(c, 64) } + want := func(f, composed string) sdkdelivery.Want { return sdkdelivery.Want{V: 1, FP: f, Composed: composed} } + have := func(f, composed string, phase sdkdelivery.Phase) sdkdelivery.Have { + return sdkdelivery.Have{V: 1, FP: f, Composed: composed, Phase: phase, Health: sdkdelivery.HealthOK} + } + // A want's key is `want..` and a have's is `.`: the same module, two buckets. + key := func(node, module string) string { + k, err := sdkdelivery.WantKey(node, module) + if err != nil { + t.Fatal(err) + } + return k + } + hkey := func(node, module string) string { + k, err := sdkdelivery.HaveKey(node, module) + if err != nil { + t.Fatal(err) + } + return k + } + for _, c := range []struct { + what string + wants map[string]sdkdelivery.Want + haves map[string]sdkdelivery.Have + current map[string]bool + }{ + {"everything had as wanted", + map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)}, + map[string]bool{"anchor": true}}, + {"a module not had yet", + map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")}, + nil, map[string]bool{"anchor": false}}, + {"an older fingerprint", + map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("b"), "d1")}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)}, + map[string]bool{"anchor": false}}, + {"the same fingerprint, an older composition", + map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d2")}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)}, + map[string]bool{"anchor": false}}, + {"still applying", + map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseApplying)}, + map[string]bool{"anchor": false}}, + {"held for a backup that failed", + map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseHeld)}, + map[string]bool{"anchor": false}}, + {"a module wanted removed that is gone", + map[string]sdkdelivery.Want{key("anchor", "web"): {V: 1, Removed: true}}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have("", "", sdkdelivery.PhaseRemoved)}, + map[string]bool{"anchor": true}}, + {"a module wanted removed that still runs", + map[string]sdkdelivery.Want{key("anchor", "web"): {V: 1, Removed: true}}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)}, + map[string]bool{"anchor": false}}, + {"one machine behind does not hold another back", + map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1"), key("laptop", "web"): want(fp("a"), "d1")}, + map[string]sdkdelivery.Have{hkey("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)}, + map[string]bool{"anchor": true, "laptop": false}}, + } { + got := currentNodes(c.wants, c.haves) + for node, want := range c.current { + if got[node] != want { + t.Errorf("%s: %s is current %v, want %v", c.what, node, got[node], want) + } + } + } + if got := currentNodes(nil, nil); len(got) != 0 { + t.Errorf("no want names a node, yet %v", got) + } +} + +// **The bus's machine is behind while its have does not name the composition carrying the user list** (ADR 0539 §15): the +// digest of the bus module's composition written now against the one its have names. +func TestTheBusMachineIsBehindUntilItsHaveNamesTheCompositionCarryingTheUserList(t *testing.T) { + have := sdkdelivery.Have{Composed: "d2", Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + for _, c := range []struct { + what string + digest string + err error + have sdkdelivery.Have + had bool + behind bool + }{ + {"its have names the composition written now", "d2", nil, have, true, false}, + {"its have names an older composition (a new user list)", "d3", nil, have, true, true}, + {"it has no have of the module yet", "d2", nil, sdkdelivery.Have{}, false, true}, + {"the composition cannot be read: not known is behind", "", errors.New("no composition"), have, true, true}, + } { + if got := listBehind(c.digest, c.err, c.have, c.had); got != c.behind { + t.Errorf("%s: behind %v, want %v", c.what, got, c.behind) + } + } +} + +// **A node nothing is wanted on that has reported is current; one that has not is not** (issue 1945): it has nothing to apply. +func TestANodeWithNoWantThatHasReportedIsCurrent(t *testing.T) { + at := time.Now() + reports := []inventory.Reported{{Node: "anchor", At: &at}, {Node: "laptop"}, {Node: "ace", At: &at}} + applyCurrent(reports, map[string]bool{"ace": false}) + got := map[string]bool{} + for _, r := range reports { + got[r.Node] = r.Current + if r.CurrentUnknown { + t.Errorf("%s: current is unknown with the haves read", r.Node) + } + } + if !got["anchor"] || got["laptop"] || got["ace"] { + t.Errorf("current by node: %v; want anchor (reported, nothing wanted) true, laptop (never reported) false, ace (behind) false", got) + } +} + +// **With the bus out of reach, whether a node is current is not known** (issue 1945): every report says so, and none says "not +// current". +func TestWithTheBusOutOfReachNoNodeIsSaidCurrentOrNot(t *testing.T) { + at := time.Now() + reports := []inventory.Reported{{Node: "anchor", At: &at, Current: true}, {Node: "laptop"}} + currentUnknown(reports) + for _, r := range reports { + if r.Current || !r.CurrentUnknown { + t.Errorf("%s: current %v, unknown %v; want not known", r.Node, r.Current, r.CurrentUnknown) + } + } +} + +// **A holder is asked once its module's have says it runs** (novox/hq ADR 0539 §9, issue 275): assigned and not applied yet, it +// is not there to answer; a machine that has said nothing holds nothing the mesh put there. +func TestAHolderIsAskedOnlyOnceItsModuleRuns(t *testing.T) { + have := func(phase sdkdelivery.Phase) sdkdelivery.Have { + return sdkdelivery.Have{Phase: phase, Health: sdkdelivery.HealthOK} + } + key := func(node, module string) string { + k, err := sdkdelivery.HaveKey(node, module) + if err != nil { + t.Fatal(err) + } + return k + } + nodeKey, err := sdkdelivery.HaveNodeKey("anchor") + if err != nil { + t.Fatal(err) + } + got := runningModules(map[string]sdkdelivery.Have{ + key("anchor", "keeper"): have(sdkdelivery.PhaseRunning), + key("anchor", "house"): have(sdkdelivery.PhaseApplying), + key("laptop", "house"): have(sdkdelivery.PhaseFailed), + nodeKey: have(sdkdelivery.PhaseRunning), + }) + now := time.Now() + if !got["anchor"].settled("keeper", now) { + t.Error("a running module's holder is not asked") + } + if got["anchor"].settled("house", now) { + t.Error("a module still applying is asked") + } + if got["laptop"].settled("house", now) || got["ace"].settled("keeper", now) { + t.Error("a failed module, or a machine that has said nothing, is asked") + } + if got["anchor"].settled("node", now) { + t.Error("the node's own have is taken for a module") + } +} diff --git a/cmd/mesh-controller/healers.go b/cmd/mesh-controller/healers.go index 7bc1c999..f9879cfc 100644 --- a/cmd/mesh-controller/healers.go +++ b/cmd/mesh-controller/healers.go @@ -12,8 +12,6 @@ import ( "sync" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" @@ -100,25 +98,6 @@ const actsInController = "controller" // test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its // event (healers_test.go). var healerRegistry = []healerRow{ - {ID: "H1", Kinds: []string{"sent-not-reported"}, - Condition: "a machine was sent a declaration and has not reported it (S2)", - Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " + - "comes back is not the declaration it was sent, or nothing comes, send it its current declaration " + - "again — what a push of that one machine does, never moving a build a policy or a plan holds back", - Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)", - Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed, - applies: appliesToAMachine, repair: repairReport}, - {ID: "H2", Kinds: []string{"stalled"}, - Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " + - "exists — or already finished — every module of every tier built or failed, and every one that " + - "rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " + - "take a transition (D14, novox/hq ADR 0239)", - Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " + - "done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " + - "reads its walk again and takes only the transition its table names", - Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)", - Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed, - applies: appliesToAStalePlan, repair: repairPlan}, {ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"}, Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " + "bus does not hold (D6, S9)", @@ -181,12 +160,8 @@ type healing struct { // The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the // bus's objects, resetting a consumer. - askReport func(ctx context.Context, node string) error - sendAgain func(ctx context.Context, node string) error assertObjects func(ctx context.Context) error resetConsumer func(stream, name string) (string, error) - // reportWait is how long H1 waits for the machine's report after asking. - reportWait time.Duration mu sync.Mutex // declined is why each condition was last passed over, so it is said once and `healers` can show it. @@ -199,26 +174,7 @@ func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Telle h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding, epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }, now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }, - reportWait: 45 * time.Second, declined: map[string]string{}} - h.askReport = func(ctx context.Context, node string) error { - if h.js == nil { - return errors.New("this controller is not on the bus") - } - return askToReport(ctx, h.js.Conn(), node) - } - h.sendAgain = func(ctx context.Context, node string) error { - // **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push - // that did not name the machine — a person's word sends a held build, a healer's does not. - held, err := heldMachines(ctx, open, []string{node}) - if err != nil { - return err - } - if why := held[node]; len(why) > 0 { - return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+ - "`push %s` sends it, on a person's word", strings.Join(why, "; "), node) - } - return sendTo(ctx, open, []string{node}) - } + declined: map[string]string{}} h.assertObjects = func(ctx context.Context) error { if h.js == nil { return errors.New("this controller is not on the bus") @@ -239,21 +195,6 @@ func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Telle return h } -// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core -// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store. -func askToReport(ctx context.Context, conn *nats.Conn, node string) error { - body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"}) - if err != nil { - return err - } - if err := conn.Publish(broker.AskReportSubject(node), body); err != nil { - return err - } - flushing, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - return conn.FlushWithContext(flushing) -} - // keep runs the healers until ctx ends. func (h *healing) keep(ctx context.Context) { tick := time.NewTicker(healEvery) @@ -537,184 +478,6 @@ func brakeObservation(acts []inventory.Heal, said string) conditions.Observation Said: strings.Join(most, ", ")} } -// --- H1 ---------------------------------------------------------------------------------------------- - -// appliesToAMachine is H1's: a machine named, its budget counted against the condition. -func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) { - if c.Subject.Machine == "" { - return "", false, "it names no machine", nil - } - return c.Key, true, "", nil -} - -// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing -// comes, send it again. -func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) { - node := c.Subject.Machine - inv := h.open.inventory - // The store's clock, as the report's time is: not the runner's, which a test moves by hand. - asked := time.Now() - askErr := h.askReport(ctx, node) - current, heard := false, false - if askErr == nil { - deadline := time.Now().Add(h.reportWait) - for { - r, found, err := lastReportOf(ctx, inv, node) - if err != nil { - return "", "", err - } - if found && r.At != nil && r.At.After(asked) { - heard, current = true, r.Current - if current { - break - } - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - return "", "", ctx.Err() - case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)): - } - } - } - if current { - return "asked " + node + "'s node-engine to say again what it last applied", - "it reported the declaration it was sent: its report had not reached the mesh", nil - } - why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb" - switch { - case askErr != nil: - why = "it could not be asked: " + askErr.Error() - case heard: - why = "it reported a declaration other than the one it was sent" - } - if err := h.sendAgain(ctx, node); err != nil { - return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed", - err - } - return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil -} - -// lastReportOf is one machine's last report beside its last send. -func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) { - reports, err := inv.LastReports(ctx) - if err != nil { - return inventory.Reported{}, false, err - } - for _, r := range reports { - if r.Node == node { - return r, true, nil - } - } - return inventory.Reported{}, false, nil -} - -// --- H2 ---------------------------------------------------------------------------------------------- - -// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty -// when it is neither, which is not H2's to repair. -func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) { - if p.Release != nil { - return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236) - } - recent, err := inv.RecentPlans(ctx, 50) - if err != nil { - return "", "", err - } - for _, newer := range recent { - if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) || - newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded || newer.Batch() { - continue - } - return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+ - "and branch", newer.ID, newer.Repository, short(newer.Commit)), nil - } - for _, tier := range p.Tiers { - for _, m := range tier { - s := p.Modules[m] - if s == nil || (s.State != "built" && s.State != "failed") { - return "", "", nil - } - if s.State == "built" && s.SentAt == nil { - u, err := inv.UpgradeOf(ctx, m) - if err != nil { - return "", "", err - } - // A recorded module whose move goes out on a machine is still the walk's to send (novox/hq ADR 0377). - if u.RollOut || len(wentOut(s.Recorded)) > 0 { - return "", "", nil - } - } - } - } - return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " + - "out was sent — nothing is left to wait on", nil -} - -// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished. -func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) { - if c.Subject.Scope == conditions.ScopeDelivery { - return appliesToAStalledDelivery(ctx, h, c) - } - p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID) - if err != nil { - return "", false, "", err - } - if !p.Open() { - return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil - } - state, _, err := planStale(ctx, h.open.inventory, p) - if err != nil { - return "", false, "", err - } - if state == "" { - return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " + - "which its own signal says", nil - } - return c.Key, true, "", nil -} - -// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set. -func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) { - if c.Subject.Scope == conditions.ScopeDelivery { - return repairDelivery(ctx, h, c) - } - inv := h.open.inventory - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return "", "", err - } - defer release() - p, err := inv.PlanByID(ctx, c.Subject.ID) - if err != nil { - return "", "", err - } - if !p.Open() { - return "closed nothing", "the plan is " + p.State + " already", nil - } - state, why, err := planStale(ctx, inv, p) - if err != nil { - return "", "", err - } - if state == "" { - return "closed nothing", "its wait is no longer superseded or finished", nil - } - p.State = state - p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why) - if state != inventory.PlanDone { - sayUnsent(&p, func(m string) bool { - u, err := inv.UpgradeOf(ctx, m) - return err == nil && u.RollOut - }) - } - if err := inv.SavePlan(ctx, &p); err != nil { - return "", "", err - } - return "closed the plan " + p.ID + " (" + state + ")", why, nil -} - // --- H3 ---------------------------------------------------------------------------------------------- // appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object. diff --git a/cmd/mesh-controller/healers_integration_test.go b/cmd/mesh-controller/healers_integration_test.go index 3a55294d..1d0f476b 100644 --- a/cmd/mesh-controller/healers_integration_test.go +++ b/cmd/mesh-controller/healers_integration_test.go @@ -5,7 +5,6 @@ package main import ( "context" "errors" - "slices" "strconv" "strings" "testing" @@ -18,94 +17,6 @@ import ( "github.com/novox/mesh-controller/internal/testbus" ) -func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil { - t.Fatal(err) - } - key := sentNotReported(t, "laptop") - - // First: the report had not reached the mesh, and asking for it brings it. - asked := 0 - h.askReport = func(ctx context.Context, node string) error { - asked++ - if node != "laptop" { - t.Errorf("asked %s", node) - } - _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied, - At: time.Now().Add(time.Second), Declared: "d2"}) - return err - } - h.tick(ctx) - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found { - t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err) - } - if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") || - c.Resolver != "healer:H1" { - t.Fatalf("after the first act: asked %d, %+v", asked, c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed || - acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 { - t.Fatalf("the act was not said as healer-acted: %+v", acts) - } - - // Within its settle, nothing more: the observation has its turn. - clock.pass(time.Minute) - h.tick(ctx) - if asked != 1 { - t.Fatalf("acted again inside the settle: asked %d", asked) - } - - // Second: the machine says nothing, so it is sent again. - clock.pass(3 * time.Minute) - sent := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(_ context.Context, node string) error { sent++; return nil } - h.tick(ctx) - if asked != 2 || sent != 1 { - t.Fatalf("the second act: asked %d, sent %d", asked, sent) - } - c, _, _ = conditionsFrom.Get(ctx, key) - if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") { - t.Fatalf("tried %+v", c.Tried) - } - - // The budget is spent: the operator's, urgent, said; and no healer touches it again. - clock.pass(4 * time.Minute) - h.tick(ctx) - c, _, _ = conditionsFrom.Get(ctx, key) - if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 || - !strings.Contains(c.Tried[2].Outcome, "budget of 2") { - t.Fatalf("not handed to the operator: %+v", c) - } - if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated { - t.Fatalf("the escalation was not said: %+v", acts) - } - clock.pass(time.Hour) - h.tick(ctx) - if asked != 2 || sent != 1 || len(told.said()) != 3 { - t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said())) - } - heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed || - heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 { - t.Fatalf("the heals kept: %+v %v", heals, err) - } - // And a heal is not a hand act: what S15 counts never sees it. - for _, x := range heals { - if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") { - t.Errorf("a heal reads as a hand act: %+v", x) - } - } -} - func TestNoHealerActsWithoutTheLease(t *testing.T) { open := aMesh(t) ctx := t.Context() @@ -123,93 +34,6 @@ func TestNoHealerActsWithoutTheLease(t *testing.T) { } } -func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - for i := 0; i < healBrakeLimit; i++ { - if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent", - Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed, - At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil { - t.Fatal(err) - } - } - sentNotReported(t, "laptop") - h.tick(ctx) // the fakes fail the test if anything acts - braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked") - if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") { - t.Fatalf("the brake was not said: %+v %v", braked, err) - } - if len(told.said()) != 0 { - t.Fatalf("a healer acted under the brake: %+v", told.said()) - } - // Past the hour of the first act, still held: it lets go an hour after the last. - clock.pass(30 * time.Minute) - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held { - t.Fatal("the brake let go before an hour had passed since the last act") - } - clock.pass(31 * time.Minute) - asked := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(context.Context, string) error { return nil } - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held { - t.Fatal("the brake held an hour after the last act") - } - if asked != 1 { - t.Fatalf("the healers did not act again after the brake let go: asked %d", asked) - } -} - -func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - created := time.Now().UTC().Add(-time.Hour) - older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}} - waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}} - newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute), - State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}} - for _, p := range []*inventory.Plan{&older, &waiting, &newer} { - if err := open.inventory.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - } - for _, id := range []string{"plan-old", "plan-other"} { - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled", - Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) - closed, err := open.inventory.PlanByID(ctx, "plan-old") - if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") || - !strings.Contains(closed.Note, "plan-new") { - t.Fatalf("the superseded plan: %+v %v", closed, err) - } - if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding { - t.Fatalf("a plan still waiting was closed: %+v", other) - } - if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf { - t.Fatalf("a plan H2 does not repair was touched: %+v", c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" { - t.Fatalf("said %+v", acts) - } - // Finished: every module built, none rolled out unsent — closed as done. - done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created, - State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}} - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") { - t.Fatalf("a finished plan reads %q %q %v", state, why, err) - } -} - func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) { open := aMesh(t) ctx := t.Context() @@ -422,28 +246,3 @@ func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T } } } - -func TestNatsAskToReportReachesTheMachine(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop")) - if err != nil { - t.Fatal(err) - } - if err := askToReport(t.Context(), conn, "laptop"); err != nil { - t.Fatal(err) - } - msg, err := asked.NextMsg(5 * time.Second) - if err != nil || !strings.Contains(string(msg.Data), "healer H1") { - t.Fatalf("the machine heard %v, %v", msg, err) - } - perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"}) - if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") || - slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") { - t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err) - } -} diff --git a/cmd/mesh-controller/healers_test.go b/cmd/mesh-controller/healers_test.go index 11829431..bd35774e 100644 --- a/cmd/mesh-controller/healers_test.go +++ b/cmd/mesh-controller/healers_test.go @@ -42,8 +42,6 @@ func raisedKinds() map[string]string { // inducedFailures are the healers seen acting in this file, by id: a row without one fails. var inducedFailures = map[string]string{ - "H1": "TestH1AsksAMachineToReportAndSendsItAgain", - "H2": "TestH2ClosesAPlanAnotherHasTakenOver", "H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget", "H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers", } @@ -170,10 +168,8 @@ func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) { open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil }) h := &healing{open: open, keeper: conditionsFrom, teller: told, epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now, - say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond, + say: func(f string, a ...any) { t.Logf(f, a...) }, declined: map[string]string{}} - h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil } - h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil } h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil } h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil } return h, told, clock diff --git a/cmd/mesh-controller/health_probes.go b/cmd/mesh-controller/health_probes.go index d9cc994c..8d2a2887 100644 --- a/cmd/mesh-controller/health_probes.go +++ b/cmd/mesh-controller/health_probes.go @@ -10,10 +10,8 @@ import ( "github.com/nats-io/nats.go" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) @@ -60,95 +58,6 @@ func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observa return nil, nil } -// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the -// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send. -func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - return machinesHealth(ctx, d, hostModule) -} - -// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them -// announced, answering the bus's discovery. -func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - return machinesHealth(ctx, d, broker.RuntimeModule) -} - -// machinesHealth judges a component on every machine running it and heard from, by its health -// definition, as the gate does — with no condition taken as the build's doing. -func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) { - inv := d.open.inventory - running, err := inv.Running(ctx, component) - if err != nil { - return nil, err - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - return nil, err - } - f, err := gatherGateFacts(ctx, d.open, coreComponent(component)) - if err != nil { - return nil, err - } - f.judged = false - heard := heardMachines(d) - var out []conditions.Observation - for _, node := range running { - if !heard[node] { - continue // a machine not heard from is S1's - } - if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound { - continue // inside the bound of a send: S2's, and the gate's - } - // What the machine did with what it was sent is not the component's health: the node-engine's is - // that it reported its current declaration at all, the node tools' that they answer. - if r := f.reports[node]; r.Current || component == broker.RuntimeModule { - now := time.Now() - r.Current, r.Outcome = true, inventory.OutcomeApplied - if r.At == nil { - r.At = &now - } - f.reports[node] = r - } - if component == hostModule { - // A node-engine reporting a build the mesh delivered, current or previous, is the version - // split's (D10), not ill health; a build the mesh did not deliver is. - f.engines[node] = deliveredOr(shelf[component], f.engines[node]) - } - h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f) - if h == healthGood { - continue - } - out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node, - Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning, - Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)}) - } - return sortedFound(out), nil -} - -// deliveredOr is the reported engine build, or the current delivered one when the report names any -// build at all: what H-engine judges is that it reports, not which. -func deliveredOr(m catalogue.Manifest, reported string) string { - if reported == "" { - return reported - } - if v := deliveredVersions(m); len(v) > 0 { - return v[0] - } - return reported -} - -// componentWords is a core component as a sentence names it. -func componentWords(component string) string { - switch component { - case hostModule: - return "node-engine" - case broker.RuntimeModule: - return "node tools" - case catalogue.ControllerSeatName: - return "controller" - } - return component -} - // probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a // request crosses it to every machine's node tools and back. func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go deleted file mode 100644 index 16ffc4d5..00000000 --- a/cmd/mesh-controller/held_back.go +++ /dev/null @@ -1,283 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq -// issue 259, ADR 0221). -// -// A named push ends by sending every other machine whose declaration differs from what it was last -// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A -// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls -// out makes every machine running it differ from the merge on, and so did a module whose plan was -// still waiting on its first machine (ADR 0218): `push ` sent all four machines the build -// that was meant to be walked through the mesh one machine at a time, and a fault in it was met -// everywhere at once. -// -// What tells the two apart is which build of each module the machine was last sent, kept with every -// send. A machine any of whose modules would move to a build its policy or an open plan holds back -// is not sent by a push that did not name it; the push says which, and why, and how to send it. - -// heldBack is why a push that did not name a machine must not send it, empty when it may. -// -// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as -// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the -// machine was last sent — including a module the machine was never sent at all. A move is held when -// the module's policy records rather than rolls out, or when an open plan has not yet sent this -// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried -// is not known, so a held upgrade cannot be told from anything else. -func heldBack(node string, modules []string, sent map[string]string, known bool, - current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string { - if !known { - return []string{"which builds it was last sent is not known — it was last sent before the " + - "mesh kept them, or sent a declaration by hand"} - } - var why []string - for _, m := range modules { - now := current[m] - was, carried := sent[m] - if carried && was == now.Commit { - continue - } - move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit)) - if !now.RollOut { - why = append(why, move+", which its upgrade policy records rather than rolls out") - continue - } - if id := planStillToSend(plans, m, node); id != "" { - why = append(why, move+", which "+id+" has not sent it yet (one machine first)") - } - } - sort.Strings(why) - return why -} - -// planStillToSend is the open plan that has a module's new build still to send this machine, or empty: -// one holding the module that has neither finished sending it nor sent it here first, and has not -// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made -// per machine. -func planStillToSend(plans []inventory.Plan, module, node string) string { - for _, p := range plans { - s, holds := p.Modules[module] - if !p.Open() || !holds { - continue - } - if s == nil { - return p.ID - } - if s.SentAt != nil || s.State == "failed" { - continue - } - first := false - for _, n := range s.First { - if n == node { - first = true - } - } - if !first { - return p.ID - } - } - return "" -} - -func fromBuild(was string, carried bool) string { - if !carried { - return "(never sent it) " - } - return "from " + buildName(was) + " " -} - -func buildName(commit string) string { - if commit == "" { - return "a build with no source" - } - return shortCommit(commit) -} - -// heldMachines reads, for each machine named, why a push that did not name it must not send it -// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left -// to the send, which says why. -func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) { - // One reading of the mesh for every machine this looks at. - ctx = withInputCache(ctx) - out := map[string][]string{} - if len(names) == 0 { - return out, nil - } - inv := open.inventory - current, err := inv.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - for _, node := range names { - plan, _, err := resolveNode(ctx, open, node) - if err != nil { - continue - } - modules := make([]string, 0, len(plan.Modules)) - for _, m := range plan.Modules { - modules = append(modules, m.Module) - } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - // A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it - // runs. - same := map[string]string{} - for m, was := range sent { - same[m] = was - if f.identical(m, was, current[m].Commit) { - same[m] = current[m].Commit - } - } - // A move under a module's own `record` that recreates nothing and moves no data goes as a module that rolls - // out goes (novox/hq ADR 0377): a walk's to send, not a person's. - here, copied := current, false - for m, was := range same { - if now, held := current[m]; held && now.OwnRecord() && !f.identical(m, was, now.Commit) && f.rolls(m, was) { - if !copied { - copied = true - here = make(map[string]inventory.CurrentBuild, len(current)) - for k, v := range current { - here[k] = v - } - } - now.RollOut = true - here[m] = now - } - } - why := heldBack(node, modules, same, known, here, plans) - // And a build no gate has seen is not carried by a send that does not judge it (ADR 0236). - for _, mv := range f.moves(node, modules, same, known, false) { - if planStillToSend(plans, mv.Module, node) == "" { - why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+ - "plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To))) - } - } - if len(why) > 0 { - sort.Strings(why) - out[node] = why - } - } - return out, nil -} - -// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it -// was not sent, that whatever else it is owed waits with it, and the command that sends it. -func sayHeld(w io.Writer, node string, why []string) { - fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+ - "holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+ - "grant from this push among it — waits with it. `push %s` sends it\n", - node, strings.Join(why, "; "), node) -} - -// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over -// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose -// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221). -// -// `handled` is every machine already sent or already said; it is not considered again. Answers the -// machines that could not be composed, as refusals. -func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool, - compose func(held context.Context, node string) (sendable, error), d delivery, holder string, - keep keeping, w io.Writer) ([]string, error) { - inv := open.inventory - var refusals []string - // Bounded by the node count: a node is marked handled the round it is considered and is never - // considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard - // against a logic error, not a real limit — if it were ever hit, that is a bug rather than a - // cascade legitimately still converging, so it is said rather than passed over in silence. - rounds := 0 - for { - would, err := wouldSend(ctx, open, nodes) - if err != nil { - return refusals, err - } - behind, err := inv.Waiting(ctx, would) - if err != nil { - return refusals, err - } - var also []string - for _, m := range behind { - if !handled[m.Node] { - also = append(also, m.Node) - } - } - if len(also) == 0 { - return refusals, nil - } - if rounds++; rounds > len(nodes) { - fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+ - "should not happen; run `push --behind` to finish\n", - rounds-1, strings.Join(also, ", ")) - return refusals, nil - } - sort.Strings(also) - held, err := heldMachines(ctx, open, also) - if err != nil { - return refusals, err - } - var sending []string - for _, name := range also { - // Every candidate this round is marked handled — the sent ones so they are not - // re-listed, the held ones because they stay held, and the refused ones so a machine - // that cannot be composed does not make the loop spin on it for ever. - handled[name] = true - if why, isHeld := held[name]; isHeld { - sayHeld(w, name, why) - continue - } - sending = append(sending, name) - } - if len(sending) == 0 { - continue - } - fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+ - "declaration since changed; sending it too\n", strings.Join(sending, ", ")) - // Tolerantly, exactly as the named send: a machine that cannot be composed is collected as - // a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066). - // Held for this round only, and after the last round's were given back, so two pushes - // cascading into each other's machines never each wait on the other. - refused, err := sendRound(ctx, open, sending, compose, d, holder, keep) - refusals = append(refusals, refused...) - if err != nil { - return refusals, err - } - } -} - -// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what -// is left out of it said, and its declaration allocated. -func composeForPush(open *stores) func(held context.Context, node string) (sendable, error) { - return func(held context.Context, node string) (sendable, error) { - // Read afresh for each machine: the send before it made and recorded things. - held = freshInput(held) - plan, _, err := resolveNode(held, open, node) - if err != nil { - return sendable{}, err - } - reportUnhostable(node, plan) - reportKept(held, plan) - declared, err := composeSend(held, open, node) - if err == nil { - reportLeftOut(node, declared) - } - return declared, err - } -} diff --git a/cmd/mesh-controller/held_back_integration_test.go b/cmd/mesh-controller/held_back_integration_test.go deleted file mode 100644 index dc0b3d7d..00000000 --- a/cmd/mesh-controller/held_back_integration_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "bytes" - "reflect" - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := time.Now().Add(-time.Hour) - aResolver(t, open, "c1c1c1c1c1", asked) - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - } - // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { - t.Fatal(err) - } - compose := composeForPush(open) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { - t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err) - } - digestOfLaptop := func() string { - sent, err := inv.Outstanding(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - return sent - } - before := digestOfLaptop() - - // The change merges; the policy is record. `push anchor` sends the anchor... - aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - // ...and its cascade leaves the laptop, saying so. - var said bytes.Buffer - d.declared = nil - refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) - if err != nil || len(refused) != 0 { - t.Fatalf("the cascade failed: %v %v", refused, err) - } - if len(d.declared) != 0 { - t.Fatalf("the cascade sent %v a build its policy records", d.declared) - } - if digestOfLaptop() != before { - t.Fatal("the laptop's last send moved: it was sent the held build") - } - for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2", - "upgrade policy records", "`push laptop` sends it"} { - if !strings.Contains(said.String(), want) { - t.Errorf("the push did not say %q:\n%s", want, said.String()) - } - } - - // Held and owed something else at once — a peer joined: still not sent, and both said: why it - // is held, and that what else it is owed waits with it. - aThirdMachine(t, open) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || digestOfLaptop() != before { - t.Fatalf("a held machine owed a consequence was sent: %v", d.declared) - } - if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") { - t.Fatalf("the push did not say both:\n%s", said.String()) - } - - // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it - // (novox/hq ADR 0236). - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { - t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) - } - // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { - t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) - } - if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" { - t.Fatalf("the new send did not record the new build: %v", builds) - } -} - -func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour)) - if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil { - t.Fatal(err) - } - compose := composeForPush(open) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - - // `push spare`, the machine just placed: the others' peers change with it. - aThirdMachine(t, open) - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - var said bytes.Buffer - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { - t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String()) - } - if strings.Contains(said.String(), "was not sent") { - t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String()) - } - - // A machine whose last send was not recorded — a declaration sent by hand — is held until named. - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - // The anchor, the hub, may still be settling from the machine placed above; the laptop is the - // question. - if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") { - t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String()) - } -} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go deleted file mode 100644 index 05e7c65a..00000000 --- a/cmd/mesh-controller/held_back_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/overlay" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its -// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything -// else that moved is still a consequence the push sends (ADR 0083). -func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) { - current := map[string]inventory.CurrentBuild{ - "resolver": {Commit: "c2c2c2c2c2"}, - "agent": {Commit: "a2", RollOut: true}, - "network": {}, - } - modules := []string{"network", "resolver", "agent"} - sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"} - - // A module whose policy records moved: held, naming it, both builds and why. - why := heldBack("laptop", modules, sent, true, current, nil) - if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") || - !strings.Contains(why[0], "upgrade policy records") { - t.Fatalf("a recorded upgrade did not hold the machine: %v", why) - } - - // Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057). - sent["resolver"] = "c2c2c2c2c2" - if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 { - t.Fatalf("a machine whose builds are all current was held: %v", why) - } - - // A module whose policy rolls out moved, and no plan holds it: sent, as before. - sent["agent"] = "a1" - if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 { - t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why) - } - - // The last send's builds are not known: held whole. - if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 || - !strings.Contains(why[0], "not known") { - t.Fatalf("a machine whose last send was not recorded was not held: %v", why) - } - - // A module the machine was never sent, under a recording policy: held, and said so. - delete(sent, "resolver") - sent["agent"] = "a2" - if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 || - !strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") { - t.Fatalf("a module never sent under a recording policy: %v", why) - } -} - -// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push -// naming some other machine must not send them for it. -func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) { - at := time.Now() - current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}} - sent := map[string]string{"agent": "a1"} - waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}} - - why := heldBack("g14", []string{"agent"}, sent, true, current, waiting) - if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") { - t.Fatalf("a machine the plan has not reached was not held: %v", why) - } - // The first machine itself was sent by the plan: not held by it. - if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 { - t.Fatalf("the plan's first machine was held: %v", why) - } - // Built but not yet sent anywhere, or not yet built: the plan has it still to send. - for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} { - plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding, - Modules: map[string]*inventory.PlanModule{"agent": s}}} - if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 { - t.Errorf("%s: not held: %v", what, why) - } - } - // Sent everywhere, failed, or a plan no longer open: the plan holds nothing back. - for what, plans := range map[string][]inventory.Plan{ - "sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}}, - "failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "failed"}}}}, - "closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built"}}}}, - } { - if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 { - t.Errorf("%s: held: %v", what, why) - } - } -} - -// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison -// reads the machine as current — and writes down which machines it declared. -type recordedDelivery struct { - inv *inventory.Inventory - declared []string -} - -func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil } - -func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { - r.declared = append(r.declared, s.node) - return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared)) -} - -// aResolver is a module built from a repository, at a commit, with something on the machine that -// says which build it is. -func aResolver(t *testing.T, open *stores, commit string, asked time.Time) { - t.Helper() - m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{ - {"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit}, - }} - if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{ - Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil { - t.Fatal(err) - } -} - -// A third machine on the private network: once it is sent, every other machine's peers change with -// it, which is a consequence a push must still send — no build moved. -func aThirdMachine(t *testing.T, open *stores) { - t.Helper() - ctx := t.Context() - record, err := open.inventory.AddNode(ctx, "spare") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil { - t.Fatal(err) - } - reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{ - {"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true}, - {"name": "systemd", "present": true}}}) - if err != nil { - t.Fatal(err) - } - var profile map[string]any - if err := json.Unmarshal(reported, &profile); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil { - t.Fatal(err) - } -} - -// The issue as it happened, against the real stores: a change merged with the policy `record`, a push -// naming the anchor, and the laptop — running the same module — left with what it had, by name. - -// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved -// for another reason is sent by a push that names someone else. diff --git a/cmd/mesh-controller/held_moves.go b/cmd/mesh-controller/held_moves.go deleted file mode 100644 index ce3fe0e6..00000000 --- a/cmd/mesh-controller/held_moves.go +++ /dev/null @@ -1,198 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). -// -// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that -// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and -// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on -// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way. -// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the -// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends -// when the machine no longer waits, whoever pushed it. -// -// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has -// its own row (S17). - -// kindMovesHeld is S24's kind: machine..moves-held. -const kindMovesHeld = "moves-held" - -// heldFacts is one machine and the moves of recorded modules that wait there for a person. -type heldFacts struct { - node string - moves []heldMove -} - -// gatherHeldMoves is, per machine that holds at least one move no person chose to hold, all its held moves, by machine. -func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) { - nodes, err := inv.Nodes(ctx) - if err != nil { - return nil, fmt.Errorf("the machines cannot be read: %w", err) - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - var out []heldFacts - for _, n := range nodes { - moves, err := heldMovesOn(ctx, inv, f, n.Name) - if err != nil { - return nil, err - } - h := heldFacts{node: n.Name, moves: moves} - if len(h.asked()) > 0 { - out = append(out, h) - } - } - return out, nil -} - -// asked are the moves no person chose to hold, and chosen the ones a person did (a module set to record with a why, the bus). -func (h heldFacts) asked() []heldMove { return h.byChoice(false) } -func (h heldFacts) chosen() []heldMove { return h.byChoice(true) } - -func (h heldFacts) byChoice(chosen bool) []heldMove { - var out []heldMove - for _, m := range h.moves { - if m.Chosen == chosen { - out = append(out, m) - } - } - return out -} - -// modules are the held moves' modules, in order, of all of them. -func (h heldFacts) modules() []string { return moduleNames(h.moves) } - -func moduleNames(moves []heldMove) []string { - out := make([]string, 0, len(moves)) - for _, m := range moves { - out = append(out, m.Module) - } - return out -} - -// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push -// names every module whose move waits there as one that may move its data: the operator's Approve is the word -// ADR 0217 asks for, and the ask says so. Offered only where no person's own hold is on the machine too: a push -// composes every recorded module at its current build, so it would send that move as well, which the ask never named -// (ADR 0259 §6), and hold the whole machine if that module's data would move. -func (h heldFacts) releaseAction() conditions.Action { - return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove, - Arguments: map[string]string{"node": h.node, "move": strings.Join(moduleNames(h.asked()), ","), "why": "", - "cause": conditions.CauseOperatorAnswer}} -} - -// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does. -func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) { - names := namesWords(moduleNames(h.asked()), 3) - headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax) - recreates, data := false, false - for _, m := range h.asked() { - switch { - case strings.Contains(m.Why, "recreates"): - recreates = true - default: - data = true - } - } - why := "would change what it runs" - switch { - case recreates && data: - why = "would restart a container or touch its data" - case recreates: - why = "would restart one of its containers" - case data: - why = "would touch its files or data" - } - explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+ - "it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+ - "has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax) - needs = "release them " + FromMeshMCPServer - if chosen := h.chosen(); len(chosen) > 0 { - // A push of the machine would send the move a person chose to hold as well: no answer is offered here. - explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+ - "it waits for you. %s is also held there by your own choice, and a push of %s would send that too, so "+ - "nothing is offered here: the push is yours at the controller's terminal.", - names, why, h.node, conditions.Capital(namesWords(moduleNames(chosen), 3)), h.node), conditions.ExplanationMax) - needs = fmt.Sprintf("push %s at the controller's terminal, once you have decided about %s.", h.node, - namesWords(moduleNames(chosen), 3)) - } - resolved = "Resolved: " + names + " is no longer waiting on " + h.node - return -} - -// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the -// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts. -func watchMovesHeld(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, h := range f.held { - var said []string - for _, m := range h.moves { - said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why)) - } - headline, explanation, needs, resolved := heldWords(h) - if c := h.chosen(); len(c) > 0 { - said = append(said, "held by a person's own choice: "+strings.Join(moduleNames(c), ", ")) - } - var actions []conditions.Action - if len(h.chosen()) == 0 { - actions = []conditions.Action{h.releaseAction()} - } - out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld, - Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, - Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them", - h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node), - Said: strings.Join(said, "; "), - Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved, - Actions: actions}) - } - return out -} - -// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a -// person: "module on machine", sorted. The plan's own record says the move waited when it was judged -// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older -// than the build now held is not named. -func planHeld(p inventory.Plan, held []heldFacts) []string { - var out []string - for _, h := range held { - for _, m := range h.moves { - s := p.Modules[m.Module] - if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) { - continue - } - out = append(out, m.Module+" on "+h.node) - } - } - sort.Strings(out) - return out -} - -// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a -// plan whose module waits for a push is not plainly "done". -func heldSaid(p inventory.Plan, held []heldFacts) string { - waiting := planHeld(p, held) - if len(waiting) == 0 { - return "" - } - return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", ")) -} - -// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before. -func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts { - held, err := gatherHeldMoves(ctx, inv) - if err != nil { - return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read - } - return held -} diff --git a/cmd/mesh-controller/held_moves_integration_test.go b/cmd/mesh-controller/held_moves_integration_test.go deleted file mode 100644 index 55b65916..00000000 --- a/cmd/mesh-controller/held_moves_integration_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func inventoryUpgrade(why string) inventory.Upgrade { return inventory.Upgrade{Why: why, By: "jochen"} } - -func mustMoveFacts(t *testing.T, rm *recordedMesh) moveFacts { - t.Helper() - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - return f -} - -// 1732/AC8 (integration): a mesh running mailu at c1 on two machines, whose registered c2 recreates a container, is read -// as two machines each holding mailu's move for a person; a bundle-only c3 on top changes nothing of it; after a machine -// is sent c2 (as a push records it) it is no longer among them, and plans that carried c2 say nothing more of it. -func TestMachinesHoldingAMoveForAPersonAreReadFromTheRecords(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - resourceOf(&c2, "imap")["image"] = "registry.invalid/imap@sha256:2" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - ctx := t.Context() - held, err := gatherHeldMoves(ctx, rm.open.inventory) - if err != nil { - t.Fatal(err) - } - if len(held) != 2 || held[0].node != "anchor" || held[1].node != "laptop" || len(held[0].moves) != 1 || - held[0].moves[0].Module != "mailu" || held[0].moves[0].To != "c2" || !strings.Contains(held[0].moves[0].Why, "recreates") { - t.Fatalf("held %+v", held) - } - if err := rm.open.inventory.RecordSent(ctx, nodeID(t, rm.open, "anchor"), "d-anchor-c2", map[string]string{"mailu": "c2"}); err != nil { - t.Fatal(err) - } - held, err = gatherHeldMoves(ctx, rm.open.inventory) - if err != nil || len(held) != 1 || held[0].node != "laptop" { - t.Fatalf("after anchor was sent c2: %+v (%v)", held, err) - } -} - -// 1732/AC9 (integration): a module a person set to record is no ask: they chose the hold. -func TestAHoldAPersonChoseIsNotAsked(t *testing.T) { - rm := aRecordedMesh(t, aMailu(false)) - c2 := aMailu(false) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventoryUpgrade("the migration week")); err != nil { - t.Fatal(err) - } - held, err := gatherHeldMoves(t.Context(), rm.open.inventory) - if err != nil || len(held) != 0 { - t.Fatalf("a machine holding only a person's own hold is asked: %+v (%v)", held, err) - } - if moves, _ := heldMovesOn(t.Context(), rm.open.inventory, mustMoveFacts(t, rm), "anchor"); len(moves) != 1 || !moves[0].Chosen { - t.Fatalf("status still names it, marked as chosen: %+v", moves) - } -} - -// 1742/AC4 (integration): a registered build naming an image of the store that the store lacks is read as a missing -// image on each machine that runs the module; one the store holds is not; and what the store holds is not asked twice. -func TestARegisteredBuildNamingAnImageTheStoreLacksIsRead(t *testing.T) { - m := aMailu(true) - resourceOf(&m, "imap")["image"] = "artifact-store://mailu/imap@sha256:" + strings.Repeat("e", 64) - rm := aRecordedMesh(t, m) - asked := 0 - lacking := func(_ context.Context, reference string) (bool, error) { asked++; return false, nil } - imagesHeld.Lock() - imagesHeld.held = map[string]bool{} - imagesHeld.Unlock() - got, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking) - if err != nil || len(got) != 2 || got[0].Module != "mailu" || got[0].Node != "anchor" || got[1].Node != "laptop" { - t.Fatalf("missing %+v (%v)", got, err) - } - holding := func(_ context.Context, reference string) (bool, error) { asked++; return true, nil } - if got, err := gatherMissingImages(t.Context(), rm.open.inventory, holding); err != nil || len(got) != 0 { - t.Fatalf("an image the store holds is missing: %+v (%v)", got, err) - } - before := asked - if _, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking); err != nil || asked != before { - t.Fatalf("an image the store was found to hold was asked about again (%d, %v)", asked-before, err) - } -} diff --git a/cmd/mesh-controller/held_moves_test.go b/cmd/mesh-controller/held_moves_test.go deleted file mode 100644 index 759837d7..00000000 --- a/cmd/mesh-controller/held_moves_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package main - -import ( - "context" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). - -// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words. -func aHeldMailu() heldFacts { - return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb", - Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}} -} - -func heldConditionOn(f *signalFacts) conditions.Condition { - o := watchMovesHeld(f)[0] - return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} -} - -// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with -// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there. -func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - f.held = []heldFacts{aHeldMailu()} - got := watchMovesHeld(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - o := got[0] - if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning || - o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" { - t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key()) - } - if len(o.Actions) != 1 { - t.Fatalf("actions: %+v", o.Actions) - } - a := o.Actions[0] - if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove || - a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { - t.Fatalf("the action is not the push of novox: %+v", a) - } - if _, takes := a.Arguments["why"]; !takes { - t.Fatal("the push takes a why, which the asker fills in with the operator's answer") - } - for _, want := range []string{"mailu", "novox"} { - if !strings.Contains(o.Headline+o.Explanation, want) { - t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation) - } - } - if !strings.Contains(o.Said, "recreates 1 of mailu's 12") { - t.Errorf("the evidence does not carry the walk's own words: %q", o.Said) - } - if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax { - t.Errorf("headline: %q", o.Headline) - } -} - -// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent. -func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - if got := watchMovesHeld(f); len(got) != 0 { - t.Fatalf("%+v", got) - } -} - -// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the -// operator's warrant calls the controller's own push with the machine and the modules, once. -func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - f.held = []heldFacts{aHeldMailu()} - c := heldConditionOn(f) - r := newAskerRig(t) - r.open = []conditions.Condition{c} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 1 || len(sent[0].Options) != 1 { - t.Fatalf("asked %+v", sent) - } - o := sent[0].Options[0] - if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" { - t.Errorf("the option is %+v", o) - } - if err := sent[0].Check(r.now); err != nil { - t.Errorf("the ask is refused as made: %v", err) - } - w := r.warrantFor(t, c.Key, "Release") - answerWith(t, r, w) - answerWith(t, r, w) - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") || - !strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) || - !strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) || - !strings.Contains(r.called[0], "chose Release") { - t.Fatalf("called %v", r.called) - } -} - -// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them. -func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) { - for _, verb := range []string{"push", "upgrade"} { - granted := false - for _, v := range broker.VerbsTheControllerActsOnAWarrant { - granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb) - } - if !granted { - t.Errorf("the controller may not act on a warrant for %s", verb) - } - } -} - -func aDoneMailuPlan() inventory.Plan { - return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone, - Tiers: [][]string{{"a"}, {"b"}, {"mailu"}}, - Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc", - Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}} -} - -// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine -// no longer holds it, or holds another build of the module, the line says nothing more. -func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) { - p := aDoneMailuPlan() - line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()}) - if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") { - t.Errorf("the line: %q", line) - } - if said := heldSaid(p, nil); said != "" { - t.Errorf("a plan whose moves were sent says %q", said) - } - other := aHeldMailu() - other.moves[0].To = "9999999ffff" - if said := heldSaid(p, []heldFacts{other}); said != "" { - t.Errorf("a plan older than the build now held says %q", said) - } - p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy} - if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" { - t.Errorf("a move the plan sent says %q", said) - } -} - -// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers -// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant. -func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) { - saved := backlogNow - t.Cleanup(func() { backlogNow = saved }) - backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person" - backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}} - got := backlogObservation() - if len(got) != 1 || len(got[0].Actions) != 1 { - t.Fatalf("raised %+v", got) - } - a := got[0].Actions[0] - if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove || - a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { - t.Fatalf("the action: %+v", a) - } - o := got[0] - c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} - r := newAskerRig(t) - r.open = []conditions.Condition{c} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" { - t.Fatalf("asked %+v", sent) - } - w := r.warrantFor(t, c.Key, "Release") - answerWith(t, r, w) - answerWith(t, r, w) - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") || - !strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") { - t.Fatalf("called %v", r.called) - } -} - -// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for -// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is. -func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) { - saved := backlogNow - t.Cleanup(func() { backlogNow = saved }) - now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding, - Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute), - Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}}, - Note: refusalNote(0, errors.New("a build waits there for its gate: novox"))} - backlogNow.held = "" - if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") { - t.Errorf("the wait is not said: %q", line) - } - backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person" - line := planLine(p, now) - if strings.Contains(line, "LATE") || strings.Contains(line, "building") || - !strings.Contains(line, "until a person releases it") { - t.Errorf("a plan held behind a failed release reads %q", line) - } -} - -// 1732/AC10: a machine that also holds a move a person chose to hold gets no Release, since a push would send that move -// too; the condition names both and says the push is the controller's terminal. A machine with only asked moves keeps it. -func TestAMachineAlsoHoldingAPersonsOwnHoldIsOfferedNoRelease(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - h := aHeldMailu() - h.moves = append(h.moves, heldMove{Module: "nftables", From: "n1aaaa", To: "n2bbbb", - Why: "waits: a person set it to record, and holds every move of it until they push", Chosen: true}) - f.held = []heldFacts{h} - got := watchMovesHeld(f) - if len(got) != 1 || len(got[0].Actions) != 0 { - t.Fatalf("%+v", got) - } - o := got[0] - text := o.Headline + o.Explanation + o.Needs + o.Summary + o.Said - for _, want := range []string{"mailu", "nftables", "terminal", "push novox"} { - if !strings.Contains(text, want) { - t.Errorf("does not say %q: %q", want, text) - } - } - w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} - if why, ok := conditions.PlainWords(w, "novox"); !ok { - t.Errorf("not plain: %s", why) - } - f.held = []heldFacts{aHeldMailu()} - if got := watchMovesHeld(f); len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["move"] != "mailu" { - t.Fatalf("a machine with only asked moves: %+v", got) - } -} diff --git a/cmd/mesh-controller/helpers_kept_integration_test.go b/cmd/mesh-controller/helpers_kept_integration_test.go new file mode 100644 index 00000000..a398f2b5 --- /dev/null +++ b/cmd/mesh-controller/helpers_kept_integration_test.go @@ -0,0 +1,109 @@ +//go:build integration + +package main + +import ( + "context" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// Helpers the integration tests of the deleted plan machinery shared with tests that remain. + +// noListener hears no node: these tests send merges only. +type noListener struct{ link.Listener } + +func becomes(t *testing.T, within time.Duration, what string, is func() bool) { + t.Helper() + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if is() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("%s did not happen within %s", what, within) +} + +// noBuildAsked once failed a test that asked build-agent for a build; nothing can now, so it says nothing. +func noBuildAsked(t *testing.T) { t.Helper() } + +func twoTiers(t *testing.T, open *stores) { + t.Helper() + for _, name := range []string{"a", "b"} { + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"}, + inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil { + t.Fatal(err) + } + } +} + +func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) { + t.Helper() + entries, err := inv.Catalogued(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.Manifest.Module == module { + return e, true + } + } + return inventory.Entry{}, false +} + +// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by +// module. +type planKeysOf struct { + values map[string]sdkdelivery.Module // by module + "@" + commit + keys map[string]builderkeys.Key +} + +func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { + if v, ok := k.values[module+"@"+commit]; ok { + return v, nil + } + return sdkdelivery.Module{}, builderkeys.ErrNoValue +} + +func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { + if key, ok := k.keys[module]; ok && key.Fingerprint == fp { + return key, nil + } + return builderkeys.Key{Module: module, Fingerprint: fp}, nil +} + +// putRunningHave writes a node-engine's have of a module: running and healthy, at a fingerprint and composition of its own. +func putRunningHave(t *testing.T, js *broker.JetStream, node, module string) { + t.Helper() + ctx := t.Context() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + kv, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: sdkdelivery.HaveBucket}) + if err != nil { + t.Fatal(err) + } + key, err := sdkdelivery.HaveKey(node, module) + if err != nil { + t.Fatal(err) + } + value, err := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64), + Composed: "sha256:" + strings.Repeat("b", 64), Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(ctx, key, value); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/mesh-controller/helpers_kept_test.go b/cmd/mesh-controller/helpers_kept_test.go new file mode 100644 index 00000000..f196680b --- /dev/null +++ b/cmd/mesh-controller/helpers_kept_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Helpers the tests of the deleted plan machinery shared with tests that remain. + +func nodeID(t *testing.T, open *stores, name string) string { + t.Helper() + n, err := open.inventory.NodeByName(context.Background(), name) + if err != nil { + t.Fatal(err) + } + return n.ID +} + +func bodyKeys(t *testing.T, s sendable) map[string]any { + t.Helper() + raw, err := s.Body() + if err != nil { + t.Fatal(err) + } + var keys map[string]any + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + return keys +} + +// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that +// announce a merge from it. +func theDeliveryClaims(context.Context) ([]inventory.Entry, error) { + return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil +} + +func claimingTheSeat(module string) inventory.Entry { + return inventory.Entry{Manifest: catalogue.Manifest{Module: module, + Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}} +} + +func aContainer(name string) catalogue.Manifest { + return catalogue.Manifest{Module: name, Version: "1", + Resources: []map[string]any{{"id": name, "type": "container", "image": name}}} +} + +// pushedBy composes a machine's declaration as the compositions are made, so its own secrets are made, and answers its digest. +// Nothing records a send: there is none (ADR 0539 §5). +func pushedBy(t *testing.T, open *stores, node string) string { + t.Helper() + ctx := t.Context() + plan, settings, err := resolveNode(ctx, open, node) + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(ctx, open, node, plan, settings) + if err != nil { + t.Fatal(err) + } + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + return digestOf(body) +} diff --git a/cmd/mesh-controller/hold.go b/cmd/mesh-controller/hold.go deleted file mode 100644 index 6c7d4d8e..00000000 --- a/cmd/mesh-controller/hold.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "context" -) - -// heldKey carries the nodes this call already holds, so an act that holds a node and then sends -// through sendTo does not wait on itself. -type heldKey struct{} - -// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR -// 0100), skipping any the context already holds, and returns a context that says it holds them. -// Release gives back only what this call took. -func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) { - already, _ := ctx.Value(heldKey{}).(map[string]bool) - var take []string - for _, n := range names { - if !already[n] { - take = append(take, n) - } - } - if len(take) == 0 { - return ctx, func() {}, nil - } - release, err := open.inventory.HoldNodes(ctx, take) - if err != nil { - return ctx, nil, err - } - held := map[string]bool{} - for n := range already { - held[n] = true - } - for _, n := range take { - held[n] = true - } - return context.WithValue(ctx, heldKey{}, held), release, nil -} diff --git a/cmd/mesh-controller/hold_test.go b/cmd/mesh-controller/hold_test.go deleted file mode 100644 index 818247f7..00000000 --- a/cmd/mesh-controller/hold_test.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "testing" - "time" -) - -// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled -// and a send that fails leave nobody waiting for the node. -func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - unmarshallable := func(context.Context, string) (sendable, error) { - return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil - } - plain := func(context.Context, string) (sendable, error) { - return sendable{Resources: []map[string]any{{"id": "x"}}}, nil - } - failing := &recordingDelivery{declareErr: errors.New("the broker went away")} - fine := &recordingDelivery{} - - for name, round := range map[string]func() error{ - "a body that cannot be marshalled": func() error { - _, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "", nil) - return err - }, - "a send that fails": func() error { - _, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "", nil) - return err - }, - } { - if err := round(); err == nil { - t.Fatalf("%s was not an error", name) - } - waiting, cancel := context.WithTimeout(ctx, 2*time.Second) - release, err := open.inventory.HoldNodes(waiting, []string{"anchor"}) - cancel() - if err != nil { - t.Fatalf("after %s the node is still held: %v", name, err) - } - release() - } -} diff --git a/cmd/mesh-controller/identity_bound_integration_test.go b/cmd/mesh-controller/identity_bound_integration_test.go index 01774c33..695cb459 100644 --- a/cmd/mesh-controller/identity_bound_integration_test.go +++ b/cmd/mesh-controller/identity_bound_integration_test.go @@ -71,11 +71,6 @@ func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) { if strings.Join(granted, ",") != "files" { t.Fatalf("the object store grants %v; files and only files fit", granted) } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) || - !strings.Contains(said, "left out of anchor's grants") { - t.Fatalf("the push does not say whom it leaves out:\n%s", said) - } // And `status` names it, and does not call the mesh well while it stands. asked, err := theThreeQuestions(ctx, open) diff --git a/cmd/mesh-controller/image_missing.go b/cmd/mesh-controller/image_missing.go index 181314da..d3c78240 100644 --- a/cmd/mesh-controller/image_missing.go +++ b/cmd/mesh-controller/image_missing.go @@ -126,12 +126,8 @@ func withoutMissingImages(ctx context.Context, open *stores, node string, s send if err != nil || address == "" { return s } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known { - sent = nil - } store := artifacts.Store{Address: address} - found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent) + found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, nil) for _, m := range found { fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image) } diff --git a/cmd/mesh-controller/issue201_integration_test.go b/cmd/mesh-controller/issue201_integration_test.go deleted file mode 100644 index a138642c..00000000 --- a/cmd/mesh-controller/issue201_integration_test.go +++ /dev/null @@ -1,181 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -func core201(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - record := func(commit string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) - b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, - Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - return b - } - before := record("c1aaaaaaaaaa", old) - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - record("c2bbbbbbbbbb", now) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - wasMoves := machineMoves - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - t.Cleanup(func() { machineMoves = wasMoves }) - return open -} - -func TestReplay201(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - - judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) - if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) - } - keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") - if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { - t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) - } - shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} - if err := applyKept(keepingRecorded(judged), open, "anchor", keeps, shelf); err != nil || shelf["mesh-controller"].Version != "1" { - t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) - } - key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() - if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { - t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) - } - - putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) - if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) - } - - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) - if _, found, _ := k.Get(ctx, key); !found { - t.Fatal("a send that kept the newer controller cleared the condition") - } - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) - if _, found, _ := k.Get(ctx, key); found { - t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") - } -} - -func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", - Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, - Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { - t.Fatal(err) - } - healer := withScope(ctx, sendScope{}) - if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("the healer's send is made: %v %v", kept, err) - } - if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) - } - if open, _ := k.Open(ctx); len(open) != 0 { - t.Fatalf("a completed put-back raised a condition: %+v", open) - } -} - -func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { - open := core201(t) - ctx := t.Context() - withConditionsInMemory(t) - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - for _, b := range []struct { - commit string - at time.Time - }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, - Asked: b.at}); err != nil { - t.Fatal(err) - } - } - if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { - t.Fatal(err) - } - // The send as it is made: the controller kept at c2, the app moved to the record. - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) - } - t.Cleanup(func() { sendRollout = wasSend }) - - moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) - if err != nil || len(sent) != 1 { - t.Fatalf("the gated send is made: %v %v", sent, err) - } - if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { - t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) - } - builds, _, err := inv.SentBuilds(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - g := &inventory.PlanGate{Carried: moves} - f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} - for _, m := range []string{"app", "mesh-controller"} { - if why, superseded := f.supersededOn(m, "anchor"); superseded { - t.Fatalf("the gate is superseded by its own send: %s", why) - } - } -} diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go deleted file mode 100644 index 7bd7085c..00000000 --- a/cmd/mesh-controller/issue201_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/lease" -) - -// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there, -// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for -// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back -// behind the build its machine was last sent: the send keeps it there, and carries the rest. - -// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed -// their gate. -func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) { - return func(module, commit string) (askedBuild, bool, error) { - at, known := table[module+"@"+commit] - b := askedBuild{asked: at} - for _, f := range failed { - if f == module+"@"+commit { - b.failedGate = true - } - } - return b, known, nil - } -} - -func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) { - old, now := t0, t0.Add(time.Minute) - table := map[string]time.Time{ - "mesh-controller@c1": old, "mesh-controller@c2": now, - "mesh-host@h1": old, "mesh-host@h2": now, - "app@a1": old, "app@a2": now, - } - sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"} - record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"} - found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table)) - if err != nil { - t.Fatal(err) - } - if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController || - found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine { - t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found) - } - if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") || - !strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") { - t.Fatalf("it says the machine, the build kept and the two ways out: %s", said) - } - - // The record forward, the same build, or a build the records do not hold: nothing kept. - for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} { - found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]}, - map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table)) - if err != nil || len(found) != 0 { - t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err) - } - } - - // A sent build that failed its gate is never ahead: the record put back before it is sent. - if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 || - found[0].module != "mesh-host" { - t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found) - } - - // A gate's own put-back carries its own modules back, and nothing else. - putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true} - if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" { - t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found) - } - if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}}, - askedIn(table)); len(found) != 2 { - t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found) - } -} - -// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the -// controller is the build before it, c1, which passed its gate when it rolled out. - -// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is -// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer -// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record, -// as a person's push does, clears the condition. - -// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was -// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that -// failed its gate is never kept. - -// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept -// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own -// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing. diff --git a/cmd/mesh-controller/issue348_integration_test.go b/cmd/mesh-controller/issue348_integration_test.go deleted file mode 100644 index a723bda0..00000000 --- a/cmd/mesh-controller/issue348_integration_test.go +++ /dev/null @@ -1,199 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -func TestReplay348(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - network := func() (conditions.Condition, bool) { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key == "machine.anchor.network" { - return c, true - } - } - return conditions.Condition{}, false - } - - // 10:58:45 — the second failing look: raised. - say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2))) - raised, ok := network() - if !ok { - t.Fatal("the resolver refusing on the control node raised nothing") - } - time.Sleep(5 * time.Millisecond) - sent := time.Now().UTC() - time.Sleep(5 * time.Millisecond) - - // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. - say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1))) - if _, ok := network(); !ok { - t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " + - "said the fault was gone while it still refused") - } - // 11:00:23 — its second look: unhealthy again, the same raising. - say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2))) - again, ok := network() - if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 { - t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count) - } - - // The gate on the control node, for a build sent after the fault began. - open2, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - f := gateFacts{judged: true, open: open2} - if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 { - t.Fatalf("a fault from before the send held the build: %+v", w) - } - - // Decided healthy: cleared. - say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0})) - if c, ok := network(); ok { - t.Fatalf("a statement that decides the names healthy left %s open", c.Key) - } -} - -// namesRefused is the control node's names part as its node-engine said it in the outage: its own -// resolver, at its own address, refusing. -func namesRefused(state string, streak int) link.NetworkPart { - p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} - if state == link.StateUnhealthy { - p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" - p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" - p.Toward = []string{"10.77.0.1"} - } - return p -} - -func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth { - state := link.StateHealthy - for _, p := range parts { - switch { - case p.State == link.StateUnhealthy: - state = link.StateUnhealthy - case p.State == link.StateUnknown && state == link.StateHealthy: - state = link.StateUnknown - } - } - return &link.NetworkHealth{State: state, Since: h0, Parts: parts} -} - -// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared -// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review -// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings. -func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) { - since := h0 - network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition { - raised := since.Add(time.Minute) - if len(gaps) > 0 { - raised = gaps[len(gaps)-1].Reopened - } - return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}, - Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised} - } - held := func(c conditions.Condition) bool { - return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true, - open: []conditions.Condition{c}}).whole != "" - } - // The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it. - flapped := network(since.Add(-49*time.Second), - conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)}) - if held(flapped) { - t.Fatal("a fault there at the send, flapping after it, held the machine") - } - // Recovered before the send, broken again after it: the send's. - recovered := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)}) - if !held(recovered) { - t.Fatal("a machine recovered at the send and broken after it passed the gate") - } - // An older gap, before the send, and the fault there at the send: not the send's. - twice := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)}, - conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)}) - if held(twice) { - t.Fatal("a fault there at the send, with an older gap, held the machine") - } - // Raised after the send, never cleared: the send's. - if !held(network(time.Time{})) { - t.Fatal("a fault raised after the send held nothing") - } - // And a module's own, through judgeHealth. - at := since.Add(2 * time.Minute) - g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", - Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, - served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, - open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ - Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing", - First: since.Add(-time.Hour), Raised: since.Add(time.Minute), - Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}} - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault there at the send: %s", why) - } - g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second) - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why) - } -} - -// Only an undecided part holds a condition that names it; a condition about another part clears, and a -// statement unknown as a whole holds every part (review of PR 179, A4). Pure. -func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}, - inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}), - "laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}), - "spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}), - "other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}), - }) - u := undecidedParts(f) - if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] || - !u["other"][link.PartTunnel] || u["other"]["*"] { - t.Fatalf("undecided: %v", u) - } - about := func(machine, said string, also ...string) conditions.Condition { - return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, - Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}} - } - for _, c := range []struct { - c conditions.Condition - held bool - }{ - {about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true}, - {about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false}, - {about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false}, - {about("spare", "route since …: no default route"), true}, - {about("hub", "anchor: names: refused", "anchor"), true}, - {about("hub", "anchor: tunnel: no handshake", "anchor"), false}, - } { - if got := heldUndecided(c.c, u); got != c.held { - t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held) - } - } -} diff --git a/cmd/mesh-controller/issue348_test.go b/cmd/mesh-controller/issue348_test.go deleted file mode 100644 index 1f8544c4..00000000 --- a/cmd/mesh-controller/issue348_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address -// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the -// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown, -// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence -// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both -// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that -// began before they were sent. - -// TestReplay348 replays the statements of the outage: the condition raised before the send is not -// cleared by the restarted engine's first, undecided statement, and the gate does not count it against -// the builds sent after it began. - -// A release walks its modules without a record per module: D10 counts what its tier names as rolling, -// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine. -func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) { - plans := []inventory.Plan{ - {ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}}, - {ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}}, - } - got := rollingModules(plans) - if !got["mesh-host"] || !got["letta"] || len(got) != 2 { - t.Fatalf("rolling: %v", got) - } -} diff --git a/cmd/mesh-controller/issue352_integration_test.go b/cmd/mesh-controller/issue352_integration_test.go deleted file mode 100644 index 1f7c399e..00000000 --- a/cmd/mesh-controller/issue352_integration_test.go +++ /dev/null @@ -1,227 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestReplay352(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) - t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) -} - -func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied - // 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil { - t.Fatal(err) - } - reports, _ := inv.LastReports(ctx) - for _, r := range reports { - if r.Node == "anchor" && r.Current { - t.Fatal("the fixture's newer send reads as reported") - } - } - gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") { - t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note) - } - if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") { - t.Fatalf("the gate does not keep what it sent: %+v", g) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} - -func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - advancePlans(ctx, b.open) // the release judges app c2 on anchor - _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) - if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { - t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) - } - if len(b.sent) != 1 { - t.Fatalf("sent %v", b.sent) - } -} - -func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - advancePlans(ctx, g.open) // anchor is sent app c2 first - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, g.open) - p := g.plan(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - // Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no - // rollback (a release may walk what the other send left waiting on anchor; that is not a put-back). - if r := p.Modules["app"].Gate.Rollback; r != "" { - t.Fatalf("a superseded judging made a rollback: %q", r) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a superseded build was marked failed") - } -} - -func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) - // Another send moves app on anchor to a build this gate does not judge. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - carried["app"] = "c3" - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, b.open) - p := b.release(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the release is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent) - } - if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found { - t.Fatal("a superseded judging kept a verdict") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} - -func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - keeper, _ := withConditionsInMemory(t) - told := &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: keeper, teller: told} - t.Cleanup(func() { doctorFrom = was }) - wasSend := sendRollout - var sent [][]string - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - build := func(id, commit, digest string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit}) - b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".", - Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle, - Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour)) - failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute)) - if versionOfBuild(previous) != strings.Repeat("1", 12) { - t.Fatalf("the build's version is %q", versionOfBuild(previous)) - } - applied, err := inv.SchemaApplied(ctx) - if err != nil || applied < 87 { - t.Fatalf("the store's schema reaches %d (%v)", applied, err) - } - // The build before never recorded what it reads: not proved, refused. - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") { - t.Fatalf("an unknown reach: %v %q", ok, why) - } - // It reads less than the store has: refused, naming both. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") { - t.Fatalf("a reach behind the store: %v %q", ok, why) - } - // Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent. - at := time.Now().Add(-5 * time.Minute) - state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at} - p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at, - State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply") - if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") { - t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note) - } - if len(sent) != 0 { - t.Fatalf("sent %v: nothing is put back", sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit { - t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit) - } - if marked, _ := inv.GateFailed(ctx, failed.ID); !marked { - t.Fatal("the failed build is not marked failed at its gate") - } - open2, _ := keeper.Open(ctx) - var found bool - for _, c := range open2 { - if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") { - found = true - } - } - if !found { - t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2) - } - // Reaching the store: allowed. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { - t.Fatalf("a build that reads the whole schema was refused: %q", why) - } - // A build with no bundle to know it by: refused. - if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") { - t.Fatalf("a build without a bundle: %v %q", ok, why) - } -} diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go deleted file mode 100644 index 86e8cea5..00000000 --- a/cmd/mesh-controller/issue352_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// novox/hq issue 352: on 2026-10-09 a release's gate on the control node read the machine's report against a -// newer send another plan had just made there — not against its own send — and failed three builds the -// machine had reported healthy ("has not reported on what it was sent"), put them back on every machine, -// to a controller older than the store's schema, and that controller then judged the newer plan's -// controller passed from the put-back build's health. - -// TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports; -// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module -// to another build supersedes the judging: no verdict, nothing put back. - -// A plan's own first send waits while a release judges the same module on that machine with another build. - -// A merge plan's judging is superseded the same way: another send moved its module on the first machine. - -// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept -// without its send reads the report against the send made last, as before. Pure. -func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490} - for _, c := range []struct { - r inventory.Reported - want bool - }{ - {inventory.Reported{Declared: "d-490", Current: false}, true}, - {inventory.Reported{Declared: "d-491", ReportedSequence: 491, Current: true}, true}, - {inventory.Reported{Declared: "d-489", ReportedSequence: 489, Current: false}, false}, - {inventory.Reported{Declared: "other", ReportedSequence: 490}, true}, // the same sequence, said by another digest - {inventory.Reported{Declared: "d-495", ReportedSequence: 495, Current: true}, true}, // the last send: this one or a later one - {inventory.Reported{Declared: "", Current: false}, false}, - } { - if got := sent.ReportsOn(c.r); got != c.want { - t.Errorf("%+v on %+v: %v", c.r, sent, got) - } - } - byDigest := inventory.SentDeclaration{Digest: "d-1"} - if !byDigest.ReportsOn(inventory.Reported{Declared: "d-1"}) || byDigest.ReportsOn(inventory.Reported{ReportedSequence: 5}) || - !byDigest.ReportsOn(inventory.Reported{Current: true}) { - t.Fatal("a send kept without a sequence is matched by its digest and by the last send alone") - } - f := gateFacts{sent: map[string]inventory.SentDeclaration{"anchor": sent}} - if !f.reportedOn("anchor", inventory.Reported{Declared: "d-490"}) || f.reportedOn("anchor", inventory.Reported{Declared: "d-1"}) { - t.Fatal("a gate that kept its send read the report against something other than it") - } - if !f.reportedOn("laptop", inventory.Reported{Current: true}) || f.reportedOn("laptop", inventory.Reported{Current: false}) { - t.Fatal("a gate that did not keep its send does not read the report against the send made last") - } - // Through the merge plan's first-machine wait too. - at := time.Now() - state := inventory.PlanModule{First: []string{"anchor"}, FirstAt: &at, - Gate: &inventory.PlanGate{Machines: []string{"anchor"}, Sent: map[string]inventory.SentDeclaration{"anchor": sent}}} - reports := []inventory.Reported{{Node: "anchor", At: &at, Outcome: inventory.OutcomeApplied, Current: false, Declared: "d-490"}} - if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting != "" || step.failed != "" { - t.Fatalf("the first machine's report on the plan's own send read as none: %+v", step) - } - reports[0].Declared = "d-480" - if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting == "" { - t.Fatalf("a report on an older send read as the plan's: %+v", step) - } -} - -// A gate judges only the build the machine was last sent: last sent another build of the module, the -// judging is superseded, whatever the machine reports. Pure. -func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - at := time.Now() - f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied, - At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, - commits: map[string]string{"mesh-controller": "e6b00e2e"}, sentBuilds: map[string]map[string]string{"anchor": {"mesh-controller": "ef26d4cb"}}} - taken := at.Add(-30 * time.Second) - f.holder = &lease.Holder{Taken: taken, Health: &lease.Health{Ready: true}} - h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f) - if h != healthSuperseded || !strings.Contains(why, "ef26d4cb") || !strings.Contains(why, "e6b00e2e") { - t.Fatalf("a controller build the machine no longer runs: %v %q", h, why) - } - f.sentBuilds["anchor"]["mesh-controller"] = "e6b00e2e" - if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded { - t.Fatalf("the build sent read as another: %q", why) - } - delete(f.sentBuilds, "anchor") - if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded { - t.Fatalf("a machine whose send is not known read as superseded: %q", why) - } - g := &inventory.PlanGate{To: "c2", Carried: []inventory.CarriedMove{{Module: "late", Node: "anchor", To: "c5"}}} - if got := judgedCommits(g, []judged{{"app", "anchor"}, {"late", "anchor"}}); got["app"] != "c2" || got["late"] != "c5" { - t.Fatalf("judged commits %v", got) - } -} - -// A move of another build of a module to a machine where a release or a plan is judging that module -// waits for that judging; the same build to that machine is already there. Pure. -func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - at := time.Now() - release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{ - Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{ - {Module: "mesh-controller", Node: "novox", From: "ef26d4cb", To: "2913c54c"}}}}} - merge := inventory.Plan{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "app": {First: []string{"anchor"}, FirstAt: &at, Commit: "c2", Gate: &inventory.PlanGate{Machines: []string{"anchor"}}}}} - f := moveFacts{plans: []inventory.Plan{release, merge}} - for _, c := range []struct { - module, node, to, want string - }{ - {"mesh-controller", "novox", "e6b00e2e", "release-1"}, // the day's case: a newer controller to the judged machine - {"mesh-controller", "novox", "2913c54c", ""}, // the same build: already there - {"mesh-controller", "ace", "2913c54c", "release-1"}, // another machine while the first is judged - {"mesh-host", "novox", "x", ""}, // a module the release does not carry - {"app", "anchor", "c2", ""}, - {"app", "anchor", "c3", "plan-1"}, - {"app", "laptop", "c2", "plan-1"}, - } { - if got := f.walkedBy(c.module, c.node, c.to); got != c.want { - t.Errorf("%s %s to %s: walked by %q, want %q", c.module, c.to, c.node, got, c.want) - } - } - release.Release.Gate.Verdict = inventory.GatePassed - merge.Modules["app"].Gate.Verdict = inventory.GatePassed - if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" || f.walkedBy("app", "laptop", "c3") != "" { - t.Fatal("a passed judging still holds a move") - } - release.Release.Gate.Verdict = "" - f.plans[0].State = inventory.PlanSuperseded - if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" { - t.Fatal("a closed release still holds a move") - } -} - -// The controller is never put back to a build that reaches less of the store's schema than the store -// has, or to one that never said what it reaches: the current build is kept, and the condition says so. - -// The lease's holder names the build the declaration told it it is, and the version stamp only without one. -func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - t.Setenv(RunningBuildVar, " ad62528c47c7 ") - if h := holderOf("x"); h.Build != "ad62528c47c7" { - t.Fatalf("the holder's build is %q", h.Build) - } - t.Setenv(RunningBuildVar, "") - if h := holderOf("x"); h.Build != version { - t.Fatalf("without a declared version the holder's build is %q", h.Build) - } - if reach, err := schemaReach(); err != nil || reach < 87 { - t.Fatalf("this build's reach is %d (%v)", reach, err) - } -} diff --git a/cmd/mesh-controller/issue459_integration_test.go b/cmd/mesh-controller/issue459_integration_test.go deleted file mode 100644 index 14c9acc0..00000000 --- a/cmd/mesh-controller/issue459_integration_test.go +++ /dev/null @@ -1,293 +0,0 @@ -//go:build integration - -package main - -import ( - "errors" - "reflect" - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("no open walk to stop: %+v", ws) - } - stopped := ws[0] - p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || - !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { - t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) - } - if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { - t.Fatalf("the stop answered %+v", said) - } - again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if err != nil || len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) - } - if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) - } - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - if err != nil || fresh.Batch() || !fresh.Open() { - t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) - } - if _, in := fresh.Modules["one"]; !in { - t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) - } - if _, in := fresh.Modules["app"]; in { - t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) - } - if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { - t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) - } - // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). - cutAt(t, open, t0.Add(5*time.Minute)) - if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { - t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) - } -} - -func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - first := catalogueMerge("cat1aaaaaaaa", "app", t0) - later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head) - hear(t, open, first, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - hear(t, open, later, t0.Add(11*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - - if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { - t.Fatal("a stop leaving out a merge the walk does not answer was taken") - } - if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { - t.Fatalf("a refused stop ended the walk: %s", p.State) - } - - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, - {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} - if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { - t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) - } - if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { - t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) - } - again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("walked again: %+v; want the other repository's merge alone", again) - } - - // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", - []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) - if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { - t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) - } - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("a batch of nothing was made: %+v", bs) - } -} - -func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) - hear(t, open, one2, t0.Add(3*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) - } - waiting := bs[0] - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { - t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) - } - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batches; want the open one alone", len(bs)) - } - b, _ := open.inventory.PlanByID(ctx, waiting.ID) - if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { - t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) - } - // Its window closed by the stop, not by the clock: cut at once. - cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) - fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) - if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { - t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) - } - var answered []string - for _, m := range fresh.Delivery.Merges { - answered = append(answered, m.Commit+">"+m.Carried) - } - if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { - t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) - } -} - -func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - sent := t0.Add(150 * time.Second) - for name, m := range walk.Modules { - if name == "app" || name == "one" { - m.First, m.FirstAt = []string{"anchor"}, &sent - } - } - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) - hear(t, open, cat2, t0.Add(3*time.Minute)) - _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { - t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) - } - p, _ := open.inventory.PlanByID(ctx, walk.ID) - if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { - t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) - } - if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { - t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) - } - if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { - t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) - } -} - -func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - _, bs := walks(t, open) - b := bs[0] - r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) - if err != nil { - t.Fatal(err) - } - if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { - t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) - } - if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped record holds %+v", left) - } - if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || - stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { - t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) - } - _ = r - cutAt(t, open, t0.Add(3*time.Minute)) - w, _ := open.inventory.PlanByID(ctx, b.ID) - if _, in := w.Modules["app"]; in || w.Batch() { - t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) - } - // A batch whose every merge is left out is stopped itself. - hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - _, bs = walks(t, open) - last := bs[0] - if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { - t.Fatalf("a batch left with nothing: %+v %v", said, err) - } - if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { - t.Fatalf("the emptied batch is %s", p.State) - } -} - -func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - ctx := t.Context() - hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) - _, bs := walks(t, open) - stale := bs[0] - moved := stale - moved.Note = "written by somebody else" - if err := open.inventory.SavePlan(ctx, &moved); err != nil { - t.Fatal(err) - } - fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, - State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} - err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, - []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) - if !errors.Is(err, inventory.ErrPlanMoved) { - t.Fatalf("a stale plan in the act: %v", err) - } - if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { - t.Fatal("the new plan was written though the act was refused") - } - if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { - t.Fatalf("the merge moved to %q though the act was refused", m.Plan) - } -} diff --git a/cmd/mesh-controller/issue459_test.go b/cmd/mesh-controller/issue459_test.go deleted file mode 100644 index 9c79df8c..00000000 --- a/cmd/mesh-controller/issue459_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one -// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges -// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them -// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it. - -// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and -// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS" -// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the -// test never ran. -func endsItsLine(t *testing.T) { - t.Cleanup(func() { fmt.Println() }) -} - -// leaved is a kept merge of a repository, made at a moment. -func leaved(repository, commit string, made time.Time) inventory.BatchedMerge { - return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made} -} - -// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it -// contains; an earlier merge of that repository and every other repository's merge are walked again. -func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) { - c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0) - c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second)) - c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second)) - o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second)) - again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"}) - if err != nil { - t.Fatal(err) - } - var walked []string - for _, m := range again { - walked = append(walked, m.Commit) - } - if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) { - t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit}, - {Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}} - if !reflect.DeepEqual(out, want) { - t.Fatalf("left out: %+v; want %+v", out, want) - } - // A merge the walk does not answer is refused, naming it: nothing ends. - if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil || - !strings.Contains(err.Error(), "ffffffffffff") { - t.Fatalf("a merge the walk does not answer was left out: %v", err) - } - if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil { - t.Fatal("a merge without its commit was taken") - } -} - -// moving is a kept merge that moved modules when it was heard. -func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge { - m.Event, _ = json.Marshal(keptMerge{Moves: modules}) - return m -} - -// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules -// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until -// none more is: a merge sharing a module with that one is left out as well. -func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) { - stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app") - reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one") - next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one") - free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three") - again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"}) - if err != nil { - t.Fatal(err) - } - if len(again) != 1 || again[0].Commit != free.Commit { - t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit}, - {Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}}, - {Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}} - if !reflect.DeepEqual(out, want) { - t.Fatalf("left out %+v; want %+v", out, want) - } -} - -// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its -// own batch moves is built, and empty tiers go. -func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) { - plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}}, - Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}} - got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"}) - if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) { - t.Fatalf("withheld %v, tiers %v", got, plan.Tiers) - } - if _, in := plan.Modules["app"]; in { - t.Fatal("a withheld module is still in the plan") - } -} - -// The seat verb passes the merges to leave out to the command. -func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) { - got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen", - "without": "novox/mesh-lab@b1,novox/mesh-lab@b2"}) - want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen", - "--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"} - if err != nil || !reflect.DeepEqual(got, want) { - t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want) - } -} - -// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped, -// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk -// that builds the other repository's module alone. - -// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it -// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only -// ended. A merge the walk does not answer is refused, and nothing ends. - -// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge -// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one -// (review of #232): its window closes, and its walk is at the newer commit, answering both. - -// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of -// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk -// that walked its merges again is refused a retry, saying why. - -// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on -// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped. - -// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody -// else since it was read refuses all of it, and no merge moves, no plan is written. - -// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent -// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent -// to; a module a merge walked again moves is sent again, and is not named. -func TestKeptNamesEveryMachineASendReached(t *testing.T) { - at := t0 - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ - "together": {SentAt: &at}, - "first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}}, - "again": {SentAt: &at}, - "unsent": {}, - }} - got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}}) - want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}}, - {Module: "together", Machines: []string{"anchor", "laptop"}}} - if !reflect.DeepEqual(got, want) { - t.Fatalf("kept %+v; want %+v", got, want) - } -} diff --git a/cmd/mesh-controller/issue529_test.go b/cmd/mesh-controller/issue529_test.go deleted file mode 100644 index 444d33ab..00000000 --- a/cmd/mesh-controller/issue529_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 529: a walk waiting because its delivery is held for a person said only "its walk waits for -// mesh-delivery's word — `plans go … --why …` starts it by hand", pointing a reader at a hand start instead of at -// the person's decision. The line names what mesh-delivery says holds the walk: the held delivery, its reason and -// `mesh-delivery.release`; or the delivery not at its turn. - -func waitingWalk(now time.Time) inventory.Plan { - return inventory.Plan{ID: "plan-1791793277221379309", Repository: "novox/mesh-sdk", Branch: "main", - Commit: "9747f3e1b1ac", State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}}, - Created: now.Add(-9 * time.Minute), Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} -} - -func TestAWalkWaitingOnAHeldDeliveryNamesTheHold(t *testing.T) { - now := time.Date(2026, 10, 12, 8, 35, 0, 0, time.UTC) - p := waitingWalk(now) - held := "novox/mesh-sdk@d41d255f28ad held since 2026-10-12T08:23:00Z: its group adr-0307-state-history's composed " + - "check did not pass for the heads that merged — a person decides: `mesh-delivery.release` (id " + - "novox/mesh-sdk@d41d255f28ad, why) lets it go, `mesh-delivery.stop` ends it" - waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@d41d255f28ad", State: "held", Walk: p.ID, Says: held}}} - line := planLineSaid(p, now, tierAtLeast, nil, waits) - for _, want := range []string{"held for a person", "composed check did not pass", "mesh-delivery.release"} { - if !strings.Contains(line, want) { - t.Fatalf("the line does not say %q: %q", want, line) - } - } - if strings.Contains(line, "starts it by hand") { - t.Fatalf("a held walk's line points at a hand start: %q", line) - } - // Told nothing — an older mesh-delivery, or none answering — it reads as before. - if line := planLineSaid(p, now, tierAtLeast, nil, nil); !strings.Contains(line, "starts it by hand") { - t.Fatalf("told nothing, the line reads %q", line) - } -} - -func TestAWalkWaitingOnADeliveryNotAtItsTurnSaysSo(t *testing.T) { - now := time.Date(2026, 10, 12, 8, 57, 0, 0, time.UTC) - p := waitingWalk(now) - waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@9747f3e1b1ac", State: "published", Walk: p.ID, - Says: "not let go yet: novox/mesh-tools@3f0c1a6cec18 (published) carried by it not at its turn"}}} - line := planLineSaid(p, now, tierAtLeast, nil, waits) - if !strings.Contains(line, "not at its turn") || !strings.Contains(line, "novox/mesh-tools@3f0c1a6cec18") { - t.Fatalf("the line reads %q", line) - } - // No hand start offered: it would skip the group's order (review of issue 529). - if strings.Contains(line, "plans go") { - t.Fatalf("a walk not at its turn points at a hand start: %q", line) - } -} - -// A walk nothing holds — the nzbgeek case, its merges' deliveries ended as nothing-to-move — is said so, and the hand -// start is the way on. -func TestAWalkHeldByNothingSaysWhyAndTheHandStart(t *testing.T) { - now := time.Date(2026, 10, 11, 20, 30, 0, 0, time.UTC) - p := waitingWalk(now) - waits := walkWaits{p.ID: {{ID: "novox/mesh-media-catalog@cdcdcdcd0000", State: "unheld", Walk: p.ID, - Says: "held by nothing: its merges' deliveries ended as nothing-to-move (novox/mesh-media-catalog@abababab0000), " + - "yet the walk moves nzbgeek, nzbget — nothing gives it its word"}}} - line := planLineSaid(p, now, tierAtLeast, nil, waits) - if !strings.Contains(line, "ended as nothing-to-move") || !strings.Contains(line, "plans go "+p.ID) { - t.Fatalf("the line reads %q", line) - } -} - -// What holds each waiting walk is asked of mesh-delivery's `stalled` with `walks`, a verb the controller's grant -// already names; only lines naming a walk are kept, by walk. -func TestWhatHoldsTheWaitingWalksIsAskedOfTheDeliveryOwner(t *testing.T) { - was := askDeliveryOwner - defer func() { askDeliveryOwner = was }() - var asked map[string]any - askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { - if verb != "stalled" { - t.Fatalf("asked %s", verb) - } - asked = args - return json.Marshal([]stalledLine{{ID: "novox/a@1", State: "held", Walk: "plan-1", Says: "held: why"}, - {ID: "novox/b@2", State: "held", Says: "a delivery past its bound, no walk"}}) - } - got := walksWaiting(t.Context(), nil, true) - if asked["walks"] != true { - t.Fatalf("stalled was asked with %v", asked) - } - if len(got) != 1 || len(got["plan-1"]) != 1 || got["plan-1"][0].Says != "held: why" { - t.Fatalf("the waiting walks are %+v", got) - } - if got := walksWaiting(t.Context(), nil, false); got != nil { - t.Fatalf("with no holder on record it asked: %+v", got) - } -} diff --git a/cmd/mesh-controller/issue611_integration_test.go b/cmd/mesh-controller/issue611_integration_test.go deleted file mode 100644 index 31aa2acb..00000000 --- a/cmd/mesh-controller/issue611_integration_test.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) { - open := windowed(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), - Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { - t.Fatal(err) - } - readme := repoMerge("one", "one1aaaaaaaa", t0) - readme.Paths = []string{"README.md"} - hear(t, open, readme, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") { - t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) - } - if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") { - t.Errorf("the walk's note does not name it: %q", w.Note) - } - - // A merge that changes the build source is walked, and says nothing unchanged. - code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute)) - hear(t, open, code, t0.Add(3*time.Minute+time.Second)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - if len(ws) != 2 { - t.Fatalf("walks: %+v", ws) - } - if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 { - t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery) - } -} diff --git a/cmd/mesh-controller/issue611_test.go b/cmd/mesh-controller/issue611_test.go deleted file mode 100644 index 2851dc55..00000000 --- a/cmd/mesh-controller/issue611_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "reflect" - "sort" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq issue 611: mesh-controller #293 changed one test file. Its delivery plan, made while another walk built -// build-agent and route-proxy (issue 606), read them whole and named them; at the cut their recorded build sources -// held none of its files, so its walk moved nothing and its record named no module. mesh-delivery could not tell -// "matched what runs" from "never built" and stopped the delivery, asking the operator about a change already on -// every machine. The walk now says, per module, that its build matched what runs (delivery.unchanged, ADR 0267 §5). - -// The shape of #293: a test file of the controller's repository. The controller, built from the root, and the two -// modules packaging the repository each recorded a build source holding none of it: each is said unchanged, with -// why; a module of another repository is not named. -func TestAMergeSaysTheModulesWhoseBuildSourceItLeftUnchanged(t *testing.T) { - const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git" - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - entries := []inventory.Entry{ - fromRepo("mesh-controller", controllerRepo, ""), - fromRepo("build-agent", catalogueRepo, "modules/build-agent"), - fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"), - fromRepo("gitea", catalogueRepo, "modules/gitea"), - } - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: []string{"go.mod", "module.json", "cmd/mesh-controller/", "internal/builder/"}}}, - "build-agent": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "cmd/mesh-builder/", "internal/builder/"}}, - {Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}}, - }, - "route-proxy": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "modules/route-proxy/cmd/route-proxy/"}}, - {Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}}, - }, - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "8f1aec0", - Paths: []string{"internal/builder/replay363_test.go"}} - if moved := reachOfMerge(m, entries, read, sharedRepositoryEdges).Moved(); len(moved) != 0 { - t.Fatalf("the fixture moves %v: a test file is in no build source", moved) - } - got := unchangedOfMerge(m, entries, read) - var names []string - for n, why := range got { - names = append(names, n) - if !strings.Contains(why, "holds none of") || !strings.Contains(why, "internal/builder/replay363_test.go") || - !strings.Contains(why, "the build the mesh holds is its build") { - t.Errorf("%s is said unchanged as %q", n, why) - } - } - if want := []string{"build-agent", "mesh-controller", "route-proxy"}; !reflect.DeepEqual(sortedOf(names), want) { - t.Fatalf("said unchanged: %v; want %v", sortedOf(names), want) - } - - // A module whose build source holds a changed file is moved, never said unchanged. - touched := m - touched.Paths = []string{"internal/builder/builder.go"} - if got := unchangedOfMerge(touched, entries, read); got["mesh-controller"] != "" || got["build-agent"] != "" { - t.Errorf("a module the merge moves is said unchanged: %v", got) - } - if got := unchangedOfMerge(touched, entries, read); got["route-proxy"] == "" { - t.Errorf("the route proxy, whose build source holds none of it, is not said unchanged: %v", got) - } - // With no build source recorded, the module is read whole and moved: nothing says it unchanged. - unsaid := map[string][]inventory.ReadRepository{"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}}} - if got := unchangedOfMerge(m, entries, unsaid); len(got) != 0 { - t.Errorf("modules with no recorded build source are said unchanged: %v", got) - } - // Files not all said: every module is moved, none said unchanged. - truncated := m - truncated.PathsTruncated = true - if got := unchangedOfMerge(truncated, entries, read); len(got) != 0 { - t.Errorf("a merge whose files were not all said says %v unchanged", got) - } -} - -// A module the walk builds anyway — another merge's, or a dependent — is not said unchanged: the modules map keeps -// meaning what the walk builds, and the two never name one module. -func TestAWalkSaysUnchangedOnlyWhatItDoesNotBuild(t *testing.T) { - unchanged := map[string]string{"build-agent": "why a", "route-proxy": "why b"} - got := unchangedLeft(unchanged, map[string]*inventory.PlanModule{"build-agent": {}}) - if !reflect.DeepEqual(got, map[string]string{"route-proxy": "why b"}) { - t.Fatalf("left %v", got) - } - if got := unchangedLeft(map[string]string{"x": "y"}, map[string]*inventory.PlanModule{"x": {}}); got != nil { - t.Fatalf("left %v; want none", got) - } -} - -// Against the store, the path a real merge takes: a merge whose module recorded a build source that holds none -// of its files is cut into a walk done with nothing to walk, whose record names the module unchanged and why. - -func sortedOf(names []string) []string { - out := append([]string{}, names...) - sort.Strings(out) - return out -} diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go deleted file mode 100644 index af1ba7d5..00000000 --- a/cmd/mesh-controller/issue648_test.go +++ /dev/null @@ -1,90 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A tier whose module is no move — made from the source every machine running it runs, so nothing is -// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the -// send that last carried that build before the walk got there, and nothing will make them report again: -// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever. -func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - - // app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates - // nothing. - for _, m := range []catalogue.Manifest{ - {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, - {Module: "web", Version: "w1"}, - } { - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil { - t.Fatal(err) - } - } - // What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is. - for _, b := range []inventory.Build{ - {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", - Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, - {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()}, - } { - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - // Each machine answered that send, applied, before the walk's tier was built. - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n, - Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1, - At: time.Now().Add(-10 * time.Second)}); err != nil { - t.Fatal(err) - } - } - built := time.Now().UTC() - p := g.plan(t) - p.Tiers = [][]string{{"app"}, {"web"}} - p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, g.open) - - if len(g.sent) != 0 { - t.Fatalf("a no-move build was sent: %v", g.sent) - } - p = g.plan(t) - if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") { - t.Fatalf("the build was not marked no move: %+v", s) - } - if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { - t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) - } - - // A machine that has not answered its last send is not waited on either: a built-by edge gates no tier - // (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop. - p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" - if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - p = g.plan(t) - if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { - t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note) - } -} diff --git a/cmd/mesh-controller/issue_204_integration_test.go b/cmd/mesh-controller/issue_204_integration_test.go deleted file mode 100644 index 666519c8..00000000 --- a/cmd/mesh-controller/issue_204_integration_test.go +++ /dev/null @@ -1,33 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { - inv := inventory.ForTest(t) - ctx, cancel := context.WithCancel(t.Context()) - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - cancel() // the sender is going away: its context is cancelled between the send and the record - body := []byte(`{"declaration":1,"resources":[]}`) - digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) - if err != nil { - // NodeByName on the cancelled context may itself refuse; the record must still be possible - // through the detached context, so look the node up again on a live one. - t.Fatalf("recording a send after cancellation failed: %v", err) - } - outstanding, err := inv.Outstanding(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if outstanding != digest || digest != digestOf(body) { - t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest) - } -} diff --git a/cmd/mesh-controller/issue_204_test.go b/cmd/mesh-controller/issue_204_test.go deleted file mode 100644 index 950fd9cb..00000000 --- a/cmd/mesh-controller/issue_204_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "context" - "testing" -) - -// A declaration composed earlier is numbered lower than one composed later, whatever order the two -// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a -// declaration composed before an assignment changed and sent after a newer one carried the higher -// number — and the machine, which refuses a lower number, took the older content as the mesh's -// newest word. Taken before the composition reads anything, the order of numbers is the order of -// compositions, and the host's refusal does what it is for. -func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) { - allot := numbered() - var composed []string - compose := func(stamp string) func(string) (sendable, error) { - return func(node string) (sendable, error) { - composed = append(composed, stamp) - return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil - } - } - // Composed first — before an assignment changed — and sent last. - stale, _ := composeEach([]string{"anchor"}, allot, compose("before")) - // Composed after the change, sent first. - fresh, _ := composeEach([]string{"anchor"}, allot, compose("after")) - - if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 { - t.Fatalf("the numbers do not follow the compositions: before=%d after=%d", - stale[0].declared.Sequence, fresh[0].declared.Sequence) - } - // Sent in the other order, the numbers do not change — so the machine that has applied the - // fresh one (2) refuses the stale one (1) when it arrives late. - if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) { - t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent") - } - if len(composed) != 2 || composed[0] != "before" { - t.Fatalf("compositions happened in an unexpected order: %v", composed) - } -} - -// The number is taken before the first read of the composition, not after it: an allotter that -// fails leaves nothing composed for that machine, and the others are still composed. -func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) { - calls := 0 - allot := func(node string) (order, error) { - if node == "anchor" { - return order{}, context.DeadlineExceeded - } - return order{sequence: 7}, nil - } - sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) { - calls++ - if node == "anchor" { - t.Fatal("anchor was composed although its number could not be taken") - } - return sendable{}, nil - }) - if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 { - t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals) - } - if len(refusals) != 1 { - t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals) - } -} - -// What was sent is written down even when the sender's context is already cancelled (issue 204): a -// controller replaced mid-send had told the machine and never recorded it, so status read "applied, -// current" over a machine that had just been sent something else. diff --git a/cmd/mesh-controller/issued.go b/cmd/mesh-controller/issued.go index 4c553807..9f1a2ece 100644 --- a/cmd/mesh-controller/issued.go +++ b/cmd/mesh-controller/issued.go @@ -315,7 +315,10 @@ func secretIssued(ctx context.Context, args []string) error { return err }) }, - send: func(machines []string) error { return sendTo(ctx, open, machines) }, + send: func(machines []string) error { + _, err := composeModules(ctx, open, machines) + return err + }, ofTheVault: filledTheNewWay, vault: theVault(ctx), } diff --git a/cmd/mesh-controller/kept_helpers.go b/cmd/mesh-controller/kept_helpers.go new file mode 100644 index 00000000..c4291b15 --- /dev/null +++ b/cmd/mesh-controller/kept_helpers.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/micro" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "regexp" + "strings" + "time" +) + +// Small helpers the plan machinery once held and other code still reads. + +// silentCallBound is how long a call to the bus may take before it is called unanswered. +var silentCallBound = 20 * time.Second + +// clipWords keeps a plain sentence within a bound, at a word. +func clipWords(s string, n int) string { + if len(s) <= n { + return s + } + cut := strings.LastIndex(s[:n-1], " ") + if cut <= 0 { + cut = n - 1 + } + return s[:cut] + "…" +} + +// orWords is s, or the words for when it is empty. +func orWords(s, none string) string { + if s == "" { + return none + } + return s +} + +func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) { + ctx, cancel := context.WithTimeout(ctx, silentCallBound) + defer cancel() + type answer struct { + v T + err error + } + done := make(chan answer, 1) + go func() { v, err := call(ctx); done <- answer{v, err} }() + select { + case a := <-done: + return a.v, a.err + case <-ctx.Done(): + var zero T + return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err()) + } +} + +func sameCommitID(a, b string) bool { + if a == "" || b == "" { + return false + } + if a == b { + return true + } + n := min(len(a), len(b)) + return n >= 7 && a[:n] == b[:n] +} + +// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its +// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads. +var wordsZone = func() *time.Location { return time.Local } + +// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces. +var wordsNow = time.Now + +var ( + planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`) + walkWd = regexp.MustCompile(`\bwalks\b`) + walkOne = regexp.MustCompile(`\bwalk\b`) +) + +// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly. +func plainRefusalOr(refusal, fallback string) string { + r := planIDWord.ReplaceAllString(refusal, "the plan") + r = walkWd.ReplaceAllString(r, "deliveries") + r = walkOne.ReplaceAllString(r, "delivery") + r = strings.TrimSpace(r) + if _, ok := conditions.Plain(r); !ok { + return fallback + } + return r +} + +// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted; +// a seam a test replaces. The holder's own refusal is an error naming it. +var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { + granted := false + for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner { + granted = granted || v.Verb == verb + } + if !granted { + return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb) + } + if conn == nil { + return nil, errors.New("this controller is not on the bus") + } + answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin) + if err != nil { + return nil, err + } + if answer.Error != "" { + return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error) + } + return unwrapToolResult(answer.Result), nil +} + +// deliveryOwnerWithin is how long the owner is given to answer. +var deliveryOwnerWithin = 10 * time.Second + +// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the +// protocol's content list holding it as text. +func unwrapToolResult(raw json.RawMessage) json.RawMessage { + var wrapped struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } + if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) { + return json.RawMessage(wrapped.Content[0].Text) + } + return raw +} + +// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules' +// tools are served where. A variable so a test needs no runtime. +var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) { + inbox := conn.NewRespInbox() + sub, err := conn.SubscribeSync(inbox) + if err != nil { + return nil, err + } + defer func() { _ = sub.Unsubscribe() }() + if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil { + return nil, fmt.Errorf("asking the bus who serves what: %w", err) + } + out := map[string]served{} + add := func(node string) served { + s, ok := out[node] + if !ok { + s = served{tools: map[string]bool{}} + } + return s + } + deadline := time.Now().Add(discoveryPatience) + for time.Now().Before(deadline) { + wait, cancel := context.WithTimeout(ctx, discoveryQuiet) + msg, err := sub.NextMsgWithContext(wait) + cancel() + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + break + } + var info micro.Info + if json.Unmarshal(msg.Data, &info) != nil { + continue + } + if info.Name == broker.RuntimeModule { + node := info.Metadata["node"] + if node == "" { + node = info.ID + } + s := add(node) + s.runtime = true + out[node] = s + } + for _, e := range info.Endpoints { + if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" { + continue + } + node := e.Metadata["node"] + if node == "" { + node = info.ID + } + s := add(node) + s.tools[e.Metadata["module"]] = true + out[node] = s + } + } + return out, nil +} + +// served is what one machine's node tools answered the bus's discovery with. +type served struct { + runtime bool + tools map[string]bool +} + +// theGraph is what the planner reads. +func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository, + []inventory.Edge, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, nil, nil, err + } + read, err := readForPlanning(ctx, inv) + if err != nil { + return nil, nil, nil, err + } + edges, err := inv.Dependencies(ctx) + if err != nil { + return nil, nil, nil, err + } + return entries, read, edges, nil +} + +// refusedWord is the word a build's failure says the reader did not know; empty when it says none. +func refusedWord(failed string) string { + if m := unknownWord.FindStringSubmatch(failed); m != nil { + return m[1] + } + return "" +} + +// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which +// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words). +var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`) diff --git a/cmd/mesh-controller/last_send_integration_test.go b/cmd/mesh-controller/last_send_integration_test.go deleted file mode 100644 index 4cb33fb7..00000000 --- a/cmd/mesh-controller/last_send_integration_test.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12, - Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", - Modules: []string{"docker", "sudo"}}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused, - Declared: "d12", Refused: "unknown field \"generation\""}); err != nil { - t.Fatal(err) - } - shown := printed(t, func() error { return showNode(ctx, open, "laptop") }) - for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40", - "docker, sudo", "unknown field \"generation\""} { - if !strings.Contains(shown, want) { - t.Errorf("node show laptop does not say %q:\n%s", want, shown) - } - } - body := printed(t, func() error { return statusFor(ctx, open, true) }) - var got struct { - LastSent []struct { - Node string `json:"node"` - Sender string `json:"sender"` - Answer string `json:"answer"` - } `json:"lastSent"` - } - if err := json.Unmarshal([]byte(body), &got); err != nil { - t.Fatalf("%v:\n%s", err, body) - } - if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" || - got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" { - t.Fatalf("status --json's last sends read %+v", got.LastSent) - } -} diff --git a/cmd/mesh-controller/last_send_test.go b/cmd/mesh-controller/last_send_test.go deleted file mode 100644 index 33c329f7..00000000 --- a/cmd/mesh-controller/last_send_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan` -// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show` -// says the last send in full and what the machine answered; and `status --json` carries it as fields, since -// the store's reader may not read the controller's tables and these verbs are the only window onto them. - -func aLastSend(answer inventory.SendAnswer) inventory.Send { - sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local) - return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57, - Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", - Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer} -} - -func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) { - at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local) - asked := answers{ - nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}}, - lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{ - Outcome: inventory.OutcomeApplied, At: &at})}, - } - shown := printed(t, func() error { return printStatus(asked) }) - var laptop, anchor []string - for _, line := range strings.Split(shown, "\n") { - fields := strings.Fields(line) - if len(fields) > 0 && fields[0] == "laptop" { - laptop = append(laptop, line) - } - if len(fields) > 0 && fields[0] == "anchor" { - anchor = append(anchor, line) - } - } - if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") || - !strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") || - !strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") { - t.Fatalf("status does not say laptop's last send in one line:\n%s", shown) - } - if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") { - t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown) - } -} - -func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) { - asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}}, - lastSentUnread: "the store went away"} - if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") { - t.Fatalf("a record of sends that could not be read is not said:\n%s", shown) - } - if asked.well() { - t.Error("a mesh whose sends could not be read is called well") - } -} - -func TestNodeShowSaysTheLastSendInFull(t *testing.T) { - at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local) - refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused, - Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n") - for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor", - "lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} { - if !strings.Contains(refused, want) { - t.Errorf("node show's last send does not say %q:\n%s", want, refused) - } - } - if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied, - At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") { - t.Errorf("an applied send is not said applied:\n%s", applied) - } - if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting, - "not reported on it yet") { - t.Errorf("a send not reported on is not said so:\n%s", waiting) - } - stale := aLastSend(inventory.SendAnswer{}) - when := at - stale.RefusedAt, stale.RefusedApplied = &when, 44 - if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") || - !strings.Contains(s, "generation 44") { - t.Errorf("a send refused for its generation is not said so:\n%s", s) - } -} - -func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) { - at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC) - asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}, - lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused, - Refused: "unknown field", At: &at})}} - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var got struct { - LastSent []map[string]any `json:"lastSent"` - } - if err := json.Unmarshal(body, &got); err != nil { - t.Fatal(err) - } - if len(got.LastSent) != 1 { - t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body) - } - s := got.LastSent[0] - if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) || - s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" || - s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" || - s["answeredAt"] == nil { - t.Fatalf("laptop's last send reads %v", s) - } - if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" { - t.Fatalf("the modules it named read %v", s["modules"]) - } -} - -// And through the store: a send recorded and refused by the machine is what `node show` and `status --json` -// read back. diff --git a/cmd/mesh-controller/lean_walk_integration_test.go b/cmd/mesh-controller/lean_walk_integration_test.go deleted file mode 100644 index bc5ebd47..00000000 --- a/cmd/mesh-controller/lean_walk_integration_test.go +++ /dev/null @@ -1,141 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, n := range []string{"desk", "laptop", "anchor"} { - if !slices.Contains(*said, "send "+n) { - t.Fatalf("%s was not sent: %v", n, *said) - } - } - if slices.Contains(*said, "judge") { - t.Fatalf("a machine was judged by its health: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} - -func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - b.broken["desk"] = true // desk fails what it is sent - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the others have no verdict of their own: %+v", p.Release) - } - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { - t.Fatalf("desk was put back: %v", sent) - } -} - -func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gates["anchor"] - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} - -func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). - if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go deleted file mode 100644 index 5f6ad330..00000000 --- a/cmd/mesh-controller/lean_walk_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package main - -import ( - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send, -// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading. -func TestAWalkEndsOnItsMachinesReports(t *testing.T) { - now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) - sent := now.Add(-time.Minute) - long := now.Add(-inventory.ApplySilentAfter - time.Minute) - at := now.Add(-10 * time.Second) - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ - "app": {Rest: map[string]inventory.SentDeclaration{ - "ace": {Digest: "d-ace", At: sent}, - "novox": {Digest: "d-novox", At: sent}, - "shanks": {Digest: "d-shanks", At: sent}, - "laptop": {Digest: "d-laptop", At: long}, - }}, - "proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}}, - }} - reports := []inventory.Reported{ - {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, - {Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"}, - {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"}, - } - waiting, failed, silent := walkReports(p, reports, now, nil) - if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) || - !slices.Equal(silent, []string{"laptop"}) { - t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent) - } - reports[2].Declared = "d-shanks" - if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 { - t.Fatalf("the walk still waits once every machine reported: %v", waiting) - } -} - -// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before -// novox/hq ADR 0360 started. -func withTheGate(t *testing.T) { - t.Helper() - was := gateFirstMachine - gateFirstMachine = true - t.Cleanup(func() { gateFirstMachine = was }) -} - -// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); -// before, the first machine was judged before the rest were sent (ADR 0353 §1). - -// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other -// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). - -// A release's send kept before it was made, and made, is judged by the controller that takes over without being -// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7). - -// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put -// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7). - -// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once -// the engine and the machine's provisioner client both said so. -func TestAWalkEndWaitsForTheProvisioner(t *testing.T) { - now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) - at := now.Add(-10 * time.Second) - plan := func(sentAt time.Time) inventory.Plan { - return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ - "ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, - "g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, - }}}} - } - reports := []inventory.Reported{ - {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, - {Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"}, - } - says := map[string]inventory.ProvReport{} - prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at} - - // (a) the engine applied, the client has not said: the end waits on both. - waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov) - if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 { - t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason. - says["ace/app"] = applied - says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at} - waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov) - if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") || - !strings.Contains(failed[0], "no value") { - t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // (c) no client report past ApplySilentAfter: named silent, never applied. - delete(says, "g14/app") - waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov) - if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) { - t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // both said it: nothing waits. - says["g14/app"] = applied - if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // (d) a send handing nothing to the provisioner is the engine's report alone, as before. - bare := plan(now.Add(-time.Minute)) - bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}} - if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) { - return inventory.ProvReport{}, false - }); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent) - } -} diff --git a/cmd/mesh-controller/lease_integration_test.go b/cmd/mesh-controller/lease_integration_test.go index 4c7e980f..4ec880ec 100644 --- a/cmd/mesh-controller/lease_integration_test.go +++ b/cmd/mesh-controller/lease_integration_test.go @@ -135,11 +135,6 @@ func TestTwoControllersOneActs(t *testing.T) { if _, err := inv.AddNode(ctx, "anchor"); err != nil { t.Fatal(err) } - saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), - State: inventory.PlanBuilding} - if err := inv.SavePlan(ctx, &saved); err == nil { - t.Fatal("B wrote a plan after losing the lease") - } store := conditions.NewInMemory() keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Epoch: func() (uint64, error) { return b.epoch(ctx) }}) diff --git a/cmd/mesh-controller/lone_merge_test.go b/cmd/mesh-controller/lone_merge_test.go deleted file mode 100644 index e5850d76..00000000 --- a/cmd/mesh-controller/lone_merge_test.go +++ /dev/null @@ -1,338 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The tests of novox/hq issue 525's window skip: a merge waits for the merge window only when a batch assembling -// holds a merge related to it, or its delivery group may have a member still to merge (ADR 0304 decision 1, ADR -// 0306 decisions 2 and 3). Walks stay one at a time. - -// A merge from no head that shares nothing, with no walk open, is cut into its walk when it is heard: no window. -func TestALoneMergeSharingNothingIsCutAtOnce(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, lone(repoMerge("one", "c1", t0)), t0) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 || !ws[0].Open() || ws[0].Commit != "c1" { - t.Fatalf("a lone merge sharing nothing: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } - if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(t0) { - t.Fatalf("its window closed at %+v; want when it was heard, %s", ws[0].Times, t0) - } -} - -// A merge related to the batch assembling joins it and waits for the window, which its coming restarts: by its -// repository, by a module in common (a dependent of what the batch moves), by an edge of the module graph between -// their modules (one that widens nothing), or by its delivery group's head. -func TestARelatedMergeJoinsTheBatchAndWaitsForTheWindow(t *testing.T) { - ctx := t.Context() - for _, c := range []string{"same repository", "a module in common", "an edge of the graph", "the same group head"} { - t.Run(c, func(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/c1") - asksWithPaths(t) - inv := open.inventory - first := repoMerge("one", "c1", t0) - second := lone(repoMerge("two", "c2", t0.Add(10*time.Second))) - why := map[string]string{"same repository": "both change novox/one", "a module in common": "both move two", - "an edge of the graph": "two depends on builder (built-by)", - "the same group head": "both are members of group " + first.Head}[c] - switch c { - case "same repository": - second = lone(repoMerge("one", "c2", t0.Add(10*time.Second))) - case "a module in common": - // two is built on one: a merge of one moves two too. - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "two", Version: "1", - Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "one"}}}}, - inventory.Source{Repository: "novox/two", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - case "an edge of the graph": - // two is built by the build machine: an edge that orders and widens nothing. - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-build-machine", Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/builder", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - first = repoMerge("builder", "c1", t0) - case "the same group head": - second = fromHead(repoMerge("two", "c2", t0.Add(10*time.Second)), first.Head) - } - hear(t, open, first, t0) - said := captured(t, func() error { - return (following{open: open}).hearMerge(ctx, second, t0.Add(10*time.Second)) - }) - if !strings.Contains(said, "("+why+")") { - t.Fatalf("the merge was not said related by %q: %s", why, said) - } - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the related merge: %d walk(s), batches %+v", len(ws), bs) - } - if bs[0].Delivery.Batch.CutAtOnce { - t.Fatal("a batch a related merge joined is cut at once") - } - cutAt(t, open, t0.Add(99*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window the related merge restarted closed: %+v", ws) - } - cutAt(t, open, t0.Add(100*time.Second)) - ws, bs = walks(t, open) - if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - }) - } -} - -// A merge from a head whose delivery group has a member still to merge waits for the window, even with no batch -// assembling, and says why; its sibling joins the same batch by their head (ADR 0306 decision 2). A merge from a -// head that names no group shares nothing and is cut at once, as one from no head is. -func TestAMergeOfAGroupWaitsEvenAlone(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - states := map[string]string{"novox/one": "ready", "novox/two": "ready", "novox/elsewhere": "ready"} - groupStates(t, "feat/x", states) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce || windowClosed(bs[0], t0) { - t.Fatalf("a group's merge alone: %d walk(s), batches %+v", len(ws), bs) - } - if words := batchWords(bs[0], t0); !strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready), novox/two (ready)") { - t.Fatalf("it reads %q", words) - } - // Its members merge: the batch is held for the last, and the window runs again from that merge. - delete(states, "novox/elsewhere") - states["novox/two"] = "merged" - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/x"), t0.Add(5*time.Second)) - if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the group's second member did not join the first: %+v", bs) - } - cutAt(t, open, t0.Add(94*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before its window closed: %+v", ws) - } - cutAt(t, open, t0.Add(95*time.Second)) - ws, bs = walks(t, open) - if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" { - t.Fatalf("at its window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } -} - -// A merge from a head that names no group, sharing nothing, with no walk open, is cut at once. -func TestAHeadedMergeInNoGroupIsCutAtOnce(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "fix/alone"), t0) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 || !ws[0].Open() { - t.Fatalf("a headed merge in no group: %d walk(s), %d batch(es)", len(ws), len(bs)) - } -} - -// When mesh-delivery cannot say a head's group, the merge waits for the window, the safe side, and its batch says -// why. -func TestAGroupNobodyCanSayWaits(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { - return nil, false, errors.New("nothing serves mesh-delivery.groups") - } - t.Cleanup(func() { deliveryGroupOf = was }) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce { - t.Fatalf("a merge whose group nobody could say: %d walk(s), batches %+v", len(ws), bs) - } - if words := batchWords(bs[0], t0); !strings.Contains(words, "mesh-delivery did not say whether group feat/x has a "+ - "member still to merge (nothing serves mesh-delivery.groups): it waits for the window, the safe side") { - t.Fatalf("it reads %q", words) - } -} - -// A member counts as still to merge only while its pull request is open, and not once this controller keeps its -// merge: a group whose other members are merged or closed makes nobody wait. -func TestAGroupsMergedOrClosedMembersMakeNobodyWait(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - members := []groupMember{{Repository: "novox/one", State: "ready"}, {Repository: "novox/two", State: "published"}, - {Repository: "novox/three", State: "stopped"}} - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return members, true, nil } - t.Cleanup(func() { deliveryGroupOf = was }) - m := fromHead(repoMerge("one", "c1", t0), "feat/x") - if why, err := groupWait(ctx, open.inventory, m, true, false); err != nil || why != "" { - t.Fatalf("the group's other members are merged or closed, yet: %q %v", why, err) - } - members = append(members, groupMember{Repository: "novox/four", State: "checked"}) - if why, _ := groupWait(ctx, open.inventory, m, true, false); !strings.Contains(why, "novox/four (checked)") { - t.Fatalf("an open member: %q", why) - } - // novox/four's merge is kept in a batch already: merged, whatever mesh-delivery says yet. - hear(t, open, fromHead(repoMerge("two", "c4", t0), "feat/x"), t0) - members[3].Repository = "novox/two" - if why, _ := groupWait(ctx, open.inventory, m, true, false); why != "" { - t.Fatalf("a member this controller keeps the merge of still waits: %q", why) - } - if got := memberOf("novox/mesh-sdk@d41d255f28ad (held)"); got != (groupMember{Repository: "novox/mesh-sdk", State: "held"}) { - t.Fatalf("a member reads %+v", got) - } -} - -// A merge related to two batches assembling folds the newer into the older, which answers every merge of both -// and names nothing twice; the folded one names the batch that took it over. -func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/a", "feat/b") - asksWithPaths(t) - ctx := t.Context() - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/a"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/b"), t0.Add(5*time.Second)) - _, bs := walks(t, open) - if len(bs) != 2 { - t.Fatalf("two merges sharing nothing: %d batch(es)", len(bs)) - } - older, newer := bs[1], bs[0] // newest first - // Of one's repository and from two's head: related to both. - hear(t, open, fromHead(repoMerge("one", "c3", t0.Add(10*time.Second)), "feat/b"), t0.Add(10*time.Second)) - _, bs = walks(t, open) - if len(bs) != 1 || bs[0].ID != older.ID || len(bs[0].Delivery.Merges) != 3 { - t.Fatalf("after a merge related to both: batches %+v; want %s answering three merges", bs, older.ID) - } - folded, err := open.inventory.PlanByID(ctx, newer.ID) - if err != nil || folded.State != inventory.PlanSuperseded || folded.Delivery == nil || - folded.Delivery.TakenOverBy != older.ID { - t.Fatalf("the newer batch reads %s %+v (%v)", folded.State, folded.Delivery, err) - } - if left, _ := open.inventory.MergesOf(ctx, newer.ID); len(left) != 0 { - t.Fatalf("the folded batch still answers %+v", left) - } - cutAt(t, open, t0.Add(100*time.Second)) - ws, bs := walks(t, open) - var cut []inventory.Plan - for _, w := range ws { - if w.State != inventory.PlanSuperseded { - cut = append(cut, w) - } - } - if len(cut) != 1 || len(bs) != 0 || cut[0].ID != older.ID || cut[0].CommitOf("novox/one") != "c3" || - cut[0].CommitOf("novox/two") != "c2" { - t.Fatalf("the folded batch's walk: %+v, %d batch(es)", cut, len(bs)) - } -} - -// With a walk open that it is related to (the same repository, issue 973), a batch cut at once still waits for -// it, said queued behind it (ADR 0276 decision 4, ADR 0407 decision 4); it is cut once that walk ended, with no -// window to wait for. -func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/c1") - asksWithPaths(t) - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("no walk open: %+v", ws) - } - first := ws[0] - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("one", "c2", heard)), heard) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID || - !bs[0].Delivery.Batch.CutAtOnce { - t.Fatalf("a lone merge during a walk: %d walk(s), batches %+v", len(ws), bs) - } - if !strings.HasPrefix(batchWords(bs[0], heard), "queued behind "+first.ID) { - t.Fatalf("it reads %q", batchWords(bs[0], heard)) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(t.Context(), &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, heard.Add(time.Second)) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { - t.Fatalf("the queued batch was not cut once the walk ended: %+v %+v", ws, bs) - } - if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(heard) { - t.Fatalf("its window closed at %+v; want when its merge was heard, %s", ws[0].Times, heard) - } -} - -// A stopped walk's other merges are walked again in a batch they are related to, and a batch of their own when -// they share nothing with the one assembling (ADR 0299, ADR 0304 decision 1): never in a batch of another -// change, whose window the stop would close. -func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - // Related to the walk by its repository (queued behind it), and to nothing the stop walks again. - hear(t, open, catalogueMerge("cat2cccccccc", "notes", t0.Add(3*time.Minute)), t0.Add(3*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) - } - other := bs[0] - _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if said.WalkedAgainBy == "" || said.WalkedAgainBy == other.ID { - t.Fatalf("walked again by %q; want a batch of its own, not %s", said.WalkedAgainBy, other.ID) - } - b, _ := open.inventory.PlanByID(ctx, other.ID) - if b.Delivery.Batch == nil || b.Delivery.Batch.Closed || len(b.Delivery.Merges) != 1 { - t.Fatalf("the other change's batch reads %+v; want it untouched by the stop", b.Delivery) - } -} - -// A catalogue batch's cut folds in every walk waiting for its delivery's word (cutBatch's carry), so what that -// walk moves is the batch's too: a lone merge related to the waiting walk joins the batch assembling and waits for -// its window, rather than being cut at once as sharing nothing with what the batch's own merges move. -func TestALoneMergeRelatedToAWalkTheCutFoldsJoinsTheBatch(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/c3") - asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Waiting() { - t.Fatalf("the walk does not wait for its word: %+v", ws) - } - hear(t, open, repoMerge("three", "c3", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - hear(t, open, lone(repoMerge("one", "c2", t0.Add(2*time.Minute+5*time.Second))), t0.Add(2*time.Minute+5*time.Second)) - _, bs := walks(t, open) - if len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 || bs[0].Delivery.Batch.CutAtOnce { - t.Fatalf("a lone merge related to the waiting walk: batches %+v", bs) - } -} diff --git a/cmd/mesh-controller/machine_network_integration_test.go b/cmd/mesh-controller/machine_network_integration_test.go deleted file mode 100644 index 0a18cf2d..00000000 --- a/cmd/mesh-controller/machine_network_integration_test.go +++ /dev/null @@ -1,266 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "os" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - openKeys := func() []string { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var keys []string - for _, c := range list { - keys = append(keys, c.Key) - } - return keys - } - healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} - rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, - Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, - {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, - Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} - - say(h0, healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("a healthy network raised %v", keys) - } - say(h0.Add(time.Minute), rewritten) - if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("the rewrite raised %v", keys) - } - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { - t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || - !strings.Contains(lines, "unhealthy resolv-conf") { - t.Fatalf("node show says:\n%s", lines) - } - say(h0.Add(2*time.Minute), healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("written back, still open: %v", keys) - } - // An engine older than the judging says no network: nothing raised, and node show says it is not known. - say(h0.Add(3*time.Minute), nil) - kept, had, _ = inv.HealthOf(ctx, "laptop") - if keys := openKeys(); len(keys) != 0 || kept.Network != nil { - t.Fatalf("an older engine: %v %+v", keys, kept.Network) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { - t.Fatalf("node show says:\n%s", lines) - } -} - -func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { - raw, err := os.ReadFile("testdata/network-drill.json") - if err != nil { - t.Fatal(err) - } - var said []link.Health - if err := json.Unmarshal(raw, &said); err != nil { - t.Fatal(err) - } - open := aMesh(t) - ctx := t.Context() - k := conditionsFrom - var raisedAt []int - for i, h := range said { - if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { - t.Fatal(err) - } - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { - t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) - } - raisedAt = append(raisedAt, i) - } - } - // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. - if len(raisedAt) != 1 || raisedAt[0] != 3 { - t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) - } -} - -func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { - for i := range parts { - if parts[i].State == "" { - parts[i].State = link.StateUnhealthy - } - parts[i].Since = h0 - } - return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} -} - -func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("want one finding, the rewrite, got %v", keys) - } - o := obs[0] - for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", - "next reconcile"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("the summary does not say %q: %s", want, o.Summary) - } - } - if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { - t.Errorf("an address or a path reached the summary: %s", o.Summary) - } - if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { - t.Errorf("the evidence or the severity is wrong: %+v", o) - } -} - -func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateHealthy), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("want the laptop's own, got %v", keys) - } - if obs[0].Severity != conditions.Warning { - t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) - } -} - -func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { - t.Fatalf("want one condition at the hub, got %v", keys) - } - o := obs[0] - if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) - f.silent["anchor"] = true - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) - } -} - -func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, noRoute), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { - t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) - } - if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { - silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", - Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("one machine alone: want its own, got %v", keys) - } - obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { - t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) - } - if !strings.Contains(obs[0].Summary, "its resolver") { - t.Fatalf("the resolver's machine is said %s", obs[0].Summary) - } -} - -func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the control node's rewritten file: %+v", obs) - } - bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} - f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the bus unreachable from the laptop: %+v", obs) - } -} - -func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("got %v", keysOf(obs)) - } -} - -// The gate: a resolver file another program rewrote waits the judging rather than failing it at the -// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own -// network fault holds the machine as a whole, as before. -func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { - since := h0 - rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, - Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} - f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} - if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { - t.Fatalf("a rewrite the send did not make: %+v", w) - } - if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || - w.on["letta"] != "" || w.waiting != "" { - t.Fatalf("a rewrite of the file a moved module owns: %+v", w) - } - own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { - t.Fatalf("the machine's own network: %+v", w) - } - unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, - Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { - t.Fatalf("a machine that cannot reach the hub: %+v", w) - } -} diff --git a/cmd/mesh-controller/machine_units_integration_test.go b/cmd/mesh-controller/machine_units_integration_test.go deleted file mode 100644 index fd9017e0..00000000 --- a/cmd/mesh-controller/machine_units_integration_test.go +++ /dev/null @@ -1,125 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - openConditions := func() map[string]conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range list { - out[c.Key] = c - } - return out - } - say := func(at time.Time, h link.Health) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil { - t.Fatal(err) - } - } - - say(h0, desktop(h0)) - got := openConditions() - machine, raised := got["machine.laptop.units"] - if !raised || len(got) != 1 { - t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got) - } - for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} { - if !strings.Contains(machine.Summary, unit) { - t.Errorf("the finding names %s: %s", unit, machine.Summary) - } - } - if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning { - t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine) - } - - say(h0.Add(time.Minute), desktop(h0.Add(time.Minute))) - got = openConditions() - module, raised := got["module.openrazer.laptop.unhealthy"] - if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") || - !strings.Contains(module.Summary, "account's own service manager") { - t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got) - } - - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 { - t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err) - } - lines := strings.Join(unitsLines(kept, had, h0), "\n") - for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it", - "openrazer-daemon.service — openrazer's"} { - if !strings.Contains(lines, want) { - t.Errorf("node show does not say %q:\n%s", want, lines) - } - } - - // An older engine says nothing of its units: nothing cleared on its word. - older := desktop(h0.Add(2 * time.Minute)) - older.Units = nil - say(h0.Add(2*time.Minute), older) - if _, still := openConditions()["machine.laptop.units"]; !still { - t.Fatal("a statement saying nothing of units cleared the machine's finding") - } - - // The operator mends the mounts and removes the leftover link; the module's unit still fails. - mended := desktop(h0.Add(3 * time.Minute)) - mended.Units.Failed = []link.FailedUnit{} - say(h0.Add(3*time.Minute), mended) - got = openConditions() - if _, still := got["machine.laptop.units"]; still { - t.Fatalf("no failed unit of the machine's own, still open: %v", got) - } - if _, still := got["module.openrazer.laptop.unhealthy"]; !still { - t.Fatalf("the module's unit still fails, and its condition stands: %v", got) - } - - // And the module's unit mended: running, nothing open. - well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute), - Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}} - say(h0.Add(4*time.Minute), well) - if got = openConditions(); len(got) != 0 { - t.Fatalf("a running service manager leaves nothing open: %v", got) - } -} - -// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged -// user unit, the module's; two mounts of the machine's own table and a unit a removed package left -// behind, the machine's. -func desktop(at time.Time) link.Health { - return link.Health{Contract: link.ReadinessContract, At: at, - Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit, - Target: "openrazer-daemon.service", State: link.StateUnhealthy, - Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}}, - Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{ - {Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0}, - {Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0}, - {Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0}, - }}} -} - -func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) { - since := h0 - units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits} - if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true, - open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 { - t.Fatalf("the machine's own failed units held a send: %+v", w) - } -} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 37e6ceaf..7f3337f1 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -63,15 +63,6 @@ func run() error { // 45 §6), so the next controller takes it at once rather than after its age. defer theLease.release() - // Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save - // that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its - // own saves. - if args[0] != "serve" { - said, done := sayingWhatAVerbKept(ctx) - inventory.PlanSaved = said - defer done() - } - switch args[0] { case "switch": return switchCommand(ctx, args[1:]) @@ -94,7 +85,8 @@ func run() error { return rehearseCommand(ctx, args[1:]) case "builds": return buildsCommand(ctx, args[1:]) - case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume": + case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume", "plans", "plan", "push", "deliver", + "delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery": return removedByName(args[0]) case "try": return tryCommand(ctx, args[1:]) @@ -113,11 +105,6 @@ func run() error { return movePlacementCommand(ctx, args[1:]) case "move-config": return moveConfigCommand(ctx, args[1:]) - case "plans": - return plansCommand(ctx, args[1:]) - case "delivery": - // The verbs the delivery's owner asks with (novox/hq ADR 0239). - return deliveryCommand(ctx, args[1:]) case "pin": return pinCommand(ctx, args[1:], true) case "unpin": @@ -140,13 +127,11 @@ func run() error { // The facts snapshot a merge check is fed (novox/hq to-be 45 §9). case "facts": return factsCommand(ctx, args[1:]) + // The bus as a planned step (novox/hq to-be 45 §8, ADR 0236). case "upgrade": return upgradeCommand(ctx, args[1:]) - // The bus as a planned step (novox/hq to-be 45 §8, ADR 0236). case "bus": return busCommand(ctx, args[1:]) - case "declare": - return declare(ctx, args[1:]) case "overlay": return overlayCommand(ctx, args[1:]) case "module": @@ -165,10 +150,6 @@ func run() error { return secretCommand(ctx, args[1:]) case "operator": return operatorCommand(ctx, args[1:]) - case "plan": - return planCommand(ctx, args[1:]) - case "push": - return pushCommand(ctx, args[1:]) case "rollout": return rolloutCommand(ctx, args[1:]) case "seats": @@ -184,7 +165,7 @@ func run() error { return handActCommand(ctx, append([]string{"list"}, args[1:]...)) // What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0). case "durations": - return durationsCommand(ctx, args[1:]) + return removedByName(args[0]) // What is wrong, and the self-check (novox/hq to-be 45 §2, §4). case "conditions": return conditionsCommand(ctx, args[1:]) @@ -379,6 +360,9 @@ func connectionName(command, verb string) string { // removedByName refuses a command the controller no longer has, naming what replaced it (novox/hq ADR 0501 §4, §5; // ADR 0429 §18; ADR 0503 §3; issue 1851, A11, and issue 1318). func removedByName(command string) error { + if why, removed := removedInRelease6(command); removed { + return &removedCommand{msg: fmt.Sprintf("`%s` no longer exists: %s", command, why)} + } replacement := map[string]string{ "build": "the controller asks no build (ADR 0501 §5): the builder builds every fingerprint the indexer writes, a " + "module is registered from its done key, and a branch is tried with `try`; a module from a repository the " + @@ -401,6 +385,8 @@ func removedByName(command string) error { "resume, or its capacity", "forget-checks": "build-agent makes no check build from R4e (ADR 0503 §3), so there is nothing to let go of", "delivery-plan": "a delivery's plan is read with `delivery-order` and the checker's verdict (ADR 0503 §3)", + "durations": "the controller measures no sends, tiers or applies since release 6 (ADR 0539 §17): the haves and wants " + + "on the bus carry what a delivery took", "delivery-check": "the checker judges every head and every delivery group (ADR 0503 §3, ADR 0527): the controller asks " + "nothing and says no verdict of a pull request", }[command] diff --git a/cmd/mesh-controller/members_file_test.go b/cmd/mesh-controller/members_file_test.go deleted file mode 100644 index 9c5b0309..00000000 --- a/cmd/mesh-controller/members_file_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "slices" - "strings" - "testing" -) - -// aLongGroup is members as JSON over 200 KB: 1666 paths of the shape of mesh-delivery #5's change (novox/hq issue 1272). -func aLongGroup(t *testing.T) string { - t.Helper() - var paths []string - for i := 0; i < 1666; i++ { - paths = append(paths, fmt.Sprintf("modules/mesh-delivery-forge-bridge/vendor/github.com/nats-io/nats-server/v2/server/certstore/certstore_windows_file_%04d.go", i)) - } - body, err := json.Marshal([]map[string]any{{"id": "novox/mesh-delivery#5", "paths": paths}, {"id": "novox/mesh-catalog#374"}}) - if err != nil { - t.Fatal(err) - } - if len(body) < 200<<10 { - t.Fatalf("the members are only %d bytes", len(body)) - } - return string(body) -} - -func readBackFromTheCommand(t *testing.T, argv []string, members string) { - t.Helper() - for _, a := range argv { - if len(a) > 64<<10 { - t.Errorf("an argument of %d bytes is handed to the command", len(a)) - } - } - i := slices.Index(argv, "--members-file") - if i < 0 { - t.Fatalf("the command does not name a members file: %.200v", argv) - } - raw, err := os.ReadFile(argv[i+1]) - if err != nil || string(raw) != members { - t.Errorf("the file does not hold the whole members list (%d bytes, %v)", len(raw), err) - } - removeMembersFile(argv) - if _, err := os.Stat(argv[i+1]); !os.IsNotExist(err) { - t.Errorf("the file is still there after the command: %v", err) - } -} - -func TestDeliveryOrderHandsALongMembersListInAFile(t *testing.T) { // 1272/AC2 - members := aLongGroup(t) - argv, err := argvFor("delivery-order", map[string]any{"members": members}) - if err != nil { - t.Fatal(err) - } - if argv[0] != "delivery" || argv[1] != "order" { - t.Errorf("the command is %.100v", strings.Join(argv, " ")) - } - readBackFromTheCommand(t, argv, members) -} - -func TestAShortMembersListIsStillTheArgumentItWas(t *testing.T) { // 1272/AC3 - members := `[{"id":"novox/a#1"},{"id":"novox/b#2"}]` - argv, err := argvFor("delivery-order", map[string]any{"members": members}) - want := []string{"delivery", "order", "--members", members} - if err != nil || !slices.Equal(argv, want) { - t.Errorf("a short list gives %v, %v, want %v", argv, err, want) - } -} - -func TestTheCommandReadsTheMembersFromTheFileAsFromTheArgument(t *testing.T) { // 1272/AC1 - members := `[{"id":"novox/a#1"}]` - argv, _ := membersFlags(strings.Repeat(" ", membersInArgument) + members) - defer removeMembersFile(argv) - if len(argv) != 2 || argv[0] != "--members-file" { - t.Fatalf("a long list gives %v", argv) - } - if st, err := os.Stat(argv[1]); err != nil || st.Mode().Perm() != 0o600 { - t.Errorf("the file is %v, %v", st, err) - } -} diff --git a/cmd/mesh-controller/merge_bus_error_integration_test.go b/cmd/mesh-controller/merge_bus_error_integration_test.go deleted file mode 100644 index 81a56d04..00000000 --- a/cmd/mesh-controller/merge_bus_error_integration_test.go +++ /dev/null @@ -1,75 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/link" -) - -// withoutJetStream is a bucket whose reads fail the way a bus answers while its streams are away. -type withoutJetStream struct { - chain.Backend - away atomic.Bool -} - -func (w *withoutJetStream) Get(ctx context.Context, key string) (chain.Entry, bool, error) { - if w.away.Load() { - return chain.Entry{}, false, nats.ErrJetStreamNotEnabled - } - return w.Backend.Get(ctx, key) -} - -func (w *withoutJetStream) Keys(ctx context.Context) ([]string, error) { - if w.away.Load() { - return nil, nats.ErrJetStreamNotEnabled - } - return w.Backend.Keys(ctx) -} - -func (w *withoutJetStream) Watch(ctx context.Context) (<-chan chain.Entry, error) { - if w.away.Load() { - return nil, nats.ErrJetStreamNotEnabled - } - return w.Backend.Watch(ctx) -} - -func (w *withoutJetStream) WatchFiltered(ctx context.Context, filter string) (<-chan chain.Entry, error) { - if w.away.Load() { - return nil, nats.ErrJetStreamNotEnabled - } - return w.Backend.WatchFiltered(ctx, filter) -} - -// **A merge heard while the bus answers "jetstream not enabled" is held, recorded nowhere, and recorded when handed -// over again** (issue 1583, mesh-catalog #404): hearMerge answers ErrTryAgain, so the bus does not acknowledge the -// announcement, and the merge joins a batch when the bus is back. -func TestAMergeHeardWhileTheBusHasNoJetStreamIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - assignments := &withoutJetStream{Backend: chain.NewMemory()} - open.inventory.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), assignments, func(context.Context) error { return nil })) - merge := catalogueMerge("cat2bbbbbbbb", "app", t0) - - assignments.away.Store(true) - err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second)) - if !errors.Is(err, link.ErrTryAgain) { - t.Fatalf("a merge heard while the bus had no jetstream answered %v, which the bus takes as done", err) - } - - assignments.away.Store(false) - hear(t, open, merge, t0.Add(10*time.Second)) - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("the merge handed over again made %d batch(es)", len(bs)) - } -} diff --git a/cmd/mesh-controller/merge_bus_error_test.go b/cmd/mesh-controller/merge_bus_error_test.go deleted file mode 100644 index cf660e25..00000000 --- a/cmd/mesh-controller/merge_bus_error_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "errors" - "testing" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/link" -) - -// **A merge the controller cannot act on is held, whatever the error** (issue 1583, after 921): the bus acknowledges an -// announcement only when hearMerge answers without ErrTryAgain, and a merge heard once is not announced again. The bus -// answering "jetstream not enabled" (mesh-catalog #404, 2026-10-21) is no outage of a kind notNow knows, and took the merge. -func TestAMergeThatFailsOnAnyErrorIsHeldAndNotTaken(t *testing.T) { - for name, cause := range map[string]error{ - "jetstream not enabled": nats.ErrJetStreamNotEnabled, - "a bucket missing": errors.New("the bucket mesh-controller_builds is not on the bus"), - "no responders": nats.ErrNoResponders, - } { - held := heldMerge(cause) - if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) { - t.Errorf("%s answers %v, which the bus takes as done", name, held) - } - } - if heldMerge(nil) != nil { - t.Error("a merge that was heard answers an error") - } - already := link.ErrTryAgain - if heldMerge(already) != already { - t.Error("a merge already held is wrapped again") - } -} diff --git a/cmd/mesh-controller/merge_on_the_bus_test.go b/cmd/mesh-controller/merge_on_the_bus_test.go deleted file mode 100644 index 3cfc5780..00000000 --- a/cmd/mesh-controller/merge_on_the_bus_test.go +++ /dev/null @@ -1,181 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "os" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// The merge path whole, against a real bus and the store (issue 973 and issue 921, novox/hq ADR 0407 decisions 1 -// and 3; ADR 0391's proof): the forge's merged event is published on the subject the controller consumes, the -// controller's server hands it over and the window keys are on the bus. Store-backed: skips without one. - -// noListener hears no node: these tests send merges only. -type noListener struct{ link.Listener } - -// theMergePathOn serves the controller's consume loop and its merge window on a test bus, and returns a way to -// publish the forge's merged event. -func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - server := link.ConnectNats(js, nil, noListener{}) - if err := server.Follows(following{open: open}); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { defer close(done); _ = server.Serve(ctx) }() - if err := startMergeWindow(ctx, open, js); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - stop() - <-done - lookingAtBatches.Wait() - setWindowClock(nil) - windowLook.Store(nil) - }) - // The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists - // is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the - // publisher waits for the consumer. - becomes(t, 10*time.Second, "the merges' consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) - return err == nil - }) - return func(m link.SourceMoved) { - t.Helper() - body, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(broker.ControllerFollows[3], body); err != nil { - t.Fatal(err) - } - } -} - -func becomes(t *testing.T, within time.Duration, what string, is func() bool) { - t.Helper() - deadline := time.Now().Add(within) - for time.Now().Before(deadline) { - if is() { - return - } - time.Sleep(50 * time.Millisecond) - } - t.Fatalf("%s did not happen within %s", what, within) -} - -func mergeKept(open *stores, repo, commit string) bool { - _, known, err := open.inventory.MergeOf(context.Background(), repo, commit) - return err == nil && known -} - -// Two merges of one repository heard while another controller holds the plans, and a rollout of that repository -// open, are both recorded once the plans are free (handed over again by the bus, never dropped), join one batch, -// and the next rollout carries both: no person released anything (issue 921). -func TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - was, wasPoll := inventory.HoldWaitFor, inventory.HoldPoll - inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 20*time.Millisecond - t.Cleanup(func() { inventory.HoldWaitFor, inventory.HoldPoll = was, wasPoll }) - // A window of a second: a merge joining a batch opens its window again (ADR 0407 decision 1), so the batch the - // two held merges make is cut when that window has closed and the rollout before it has ended, not before. - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - publish := theMergePathOn(t, open) - ctx := t.Context() - - publish(repoMerge("one", "c1", time.Now())) - becomes(t, 15*time.Second, "the first rollout opening", func() bool { - ws, _ := walks(t, open) - return len(ws) == 1 && ws[0].Open() - }) - ws, _ := walks(t, open) - first := ws[0] - - release, err := open.inventory.HoldPlans(ctx, true) // the other controller works the plans - if err != nil { - t.Fatal(err) - } - publish(repoMerge("one", "c2", time.Now())) - publish(repoMerge("one", "c3", time.Now())) - time.Sleep(2500 * time.Millisecond) // both met the hold and were given back to the bus - if mergeKept(open, "novox/one", "c2") || mergeKept(open, "novox/one", "c3") { - t.Fatal("a merge was recorded while another holder had the plans") - } - release() - becomes(t, 30*time.Second, "both merges recorded after the hold", func() bool { - return mergeKept(open, "novox/one", "c2") && mergeKept(open, "novox/one", "c3") - }) - - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - // The rollout ended: the look that cuts what waited behind it, once the batch's window is closed too. - var bs []inventory.Plan - becomes(t, 15*time.Second, "the next rollout being cut with both merges", func() bool { - advanceHeld(ctx, open) - ws, bs = walks(t, open) - return len(bs) == 0 && len(ws) == 2 - }) - if ws[0].CommitOf("novox/one") != "c3" || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the next rollout does not carry both merges: commit %q, merges %+v", ws[0].CommitOf("novox/one"), - ws[0].Delivery.Merges) - } -} - -// A batch's window closes by the expiry of its key on the bus, heard by the watcher, and the batch is cut with -// no cutter and no call to cut (ADR 0407 decision 1): a window of a second, a merge that waits for it. -func TestABatchWindowClosesByTheExpiryOfItsKey(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - everyHeadWaits(t) - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - publish := theMergePathOn(t, open) - - start := time.Now() - publish(repoMerge("one", "c1", start)) - becomes(t, 10*time.Second, "the merge waiting in a batch", func() bool { - _, bs := walks(t, open) - return len(bs) == 1 && bs[0].State == inventory.PlanAssembling - }) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("the batch was cut before its window closed: %+v", ws) - } - becomes(t, 15*time.Second, "the batch being cut by its key's expiry", func() bool { - ws, bs := walks(t, open) - return len(ws) == 1 && len(bs) == 0 - }) - if took := time.Since(start); took < time.Second { - t.Fatalf("the batch was cut after %s, before its window of a second was up", took) - } -} diff --git a/cmd/mesh-controller/merge_unsaved_integration_test.go b/cmd/mesh-controller/merge_unsaved_integration_test.go deleted file mode 100644 index 095930b9..00000000 --- a/cmd/mesh-controller/merge_unsaved_integration_test.go +++ /dev/null @@ -1,71 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/link" -) - -// refusing is a bucket that refuses to write while it is told to. -type refusing struct { - chain.Backend - refuse atomic.Bool -} - -func (r *refusing) Create(ctx context.Context, key string, value []byte) (uint64, error) { - if r.refuse.Load() { - return 0, nats.ErrTimeout - } - return r.Backend.Create(ctx, key, value) -} - -func (r *refusing) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { - if r.refuse.Load() { - return 0, nats.ErrTimeout - } - return r.Backend.Update(ctx, key, value, revision) -} - -// **A merge whose plan cannot be saved is held, recorded nowhere, and recorded when it is handed over again** (ADR 0407 -// decision 3, issue 1092): hearMerge answers ErrTryAgain — the bus leaves the announcement unacknowledged and hands it -// over again — and the merge is in no record until the plan holding it is written. -func TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - rollouts := &refusing{Backend: chain.NewMemory()} - open.inventory.UseChain(chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })) - merge := catalogueMerge("cat1aaaaaaaa", "app", t0) - - rollouts.refuse.Store(true) - err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second)) - if !errors.Is(err, link.ErrTryAgain) { - t.Fatalf("a merge whose plan could not be saved answered %v, which the bus takes as done", err) - } - if _, known, err := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); err != nil || known { - t.Fatalf("the merge is kept though its plan was not: %v %v", known, err) - } - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("a batch was kept though its write failed: %+v", bs) - } - - rollouts.refuse.Store(false) - hear(t, open, merge, t0.Add(10*time.Second)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge handed over again made %d batch(es)", len(bs)) - } - if m, known, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); !known || m.Plan != bs[0].ID { - t.Fatalf("the merge is %+v %v, and the batch %s", m, known, bs[0].ID) - } -} diff --git a/cmd/mesh-controller/merge_unsaved_test.go b/cmd/mesh-controller/merge_unsaved_test.go deleted file mode 100644 index c967060f..00000000 --- a/cmd/mesh-controller/merge_unsaved_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "errors" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// **A merge whose plan could not be saved is held, never taken** (novox/hq ADR 0407 decision 3, issue 1092): the merge is -// carried until its plan is saved, and the bus acknowledges the announcement only when the controller answers without -// ErrTryAgain (link.Server.sourceMoved), so a failed save of any kind must answer it. -func TestAFailureToSaveAMergesPlanIsHeldAndNotTaken(t *testing.T) { - cause := errors.New("the plan for novox/mesh-catalog is not written: the plan was written by somebody else since it was read") - held := unsaved(cause) - if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) { - t.Fatalf("a failed save answers %v", held) - } - if unsaved(nil) != nil { - t.Fatal("a save that worked answers an error") - } - // Not the same as notNow, which holds a merge only for a store that is away. - if errors.Is(notNow(cause), link.ErrTryAgain) { - t.Fatal("notNow holds every failure; the premise of this test is gone") - } -} diff --git a/cmd/mesh-controller/merge_window_bus.go b/cmd/mesh-controller/merge_window_bus.go deleted file mode 100644 index bd721841..00000000 --- a/cmd/mesh-controller/merge_window_bus.go +++ /dev/null @@ -1,204 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The merge window's clock is on the bus (novox/hq ADR 0407 decision 1, issue 973): an open batch has a key -// `.quiet` that lives the window and is made again by each merge into the batch, and a key `.most` -// made once when the batch opens that lives the window's most. The first of the two to expire closes the batch, -// and the watcher hears it: no timer polls the batches. The store still keeps the times (ClosesAt, AtMost) as the -// record of when a window closed, which `plans` reads; the keys are what wakes the controller. - -// windowClock is the bus's side of the merge window. -type windowClock interface { - // Arm makes a batch's keys: its quiet key again, to live quiet; its most key if it has none, to live most. - Arm(ctx context.Context, batch string, quiet, most time.Duration) error - // Forget takes a batch's keys away: it was cut, or closed, and nothing is to wake the controller for it. - Forget(ctx context.Context, batch string) -} - -// windowLook is the look a window's expiry takes: the watcher's, and the fallback timer's. -var windowLook atomic.Pointer[func(batch string)] - -var ( - windowMu sync.RWMutex - windowKeyed windowClock // nil in a process that is not the serving controller -) - -// setWindowClock makes clock the merge window's; nil takes it away. -func setWindowClock(c windowClock) { - windowMu.Lock() - defer windowMu.Unlock() - windowKeyed = c -} - -func windowClockOf() windowClock { - windowMu.RLock() - defer windowMu.RUnlock() - return windowKeyed -} - -// windowDeadlines are how long a batch's two keys are to live from now: what is left of its window and of its -// most, each at least a second (the least a key can live). Not armed for a batch whose window is closed already -// (by a stop, by its time, or because it is cut at once) or that is no batch. -func windowDeadlines(b inventory.Plan, now time.Time) (quiet, most time.Duration, armed bool) { - if b.Delivery == nil || b.Delivery.Batch == nil || windowClosed(b, now) || len(b.Delivery.Batch.Holds) > 0 { - return 0, 0, false // a held batch has no clock: its group's last member closes it (ADR 0418 clause 3) - } - // Rounded up to a whole second, at least one: the server rounds a key's time to live down to whole seconds, so - // a fraction would close the key before the record's time and the look would cut nothing. - least := func(d time.Duration) time.Duration { - if d < time.Second { - return time.Second - } - return (d + time.Second - 1) / time.Second * time.Second - } - w := b.Delivery.Batch - return least(w.ClosesAt.Sub(now)), least(w.AtMost.Sub(now)), true -} - -// armWindow puts a batch's clock on the bus after its record was kept. Where the bus will not take it the window -// closes by a timer of this batch alone, once, as long as is left: never silently never. -func armWindow(ctx context.Context, b inventory.Plan, now time.Time) { - c := windowClockOf() - quiet, most, armed := windowDeadlines(b, now) - if c == nil || !armed { - return - } - if err := c.Arm(ctx, b.ID, quiet, most); err != nil { - fmt.Printf("batches: the window of %s could not be put on the bus (%v); it closes by a timer of its own\n", b.ID, err) - if look := windowLook.Load(); look != nil { - time.AfterFunc(quiet, func() { (*look)(b.ID) }) - } - } -} - -// busWindow is the clock on the controller's bucket. -type busWindow struct { - kv jetstream.KeyValue -} - -// newBusWindow opens the bucket and starts the watcher, which names each batch whose key expired to look until -// the context ends. Only expiry wakes it: a key taken away by Forget or made again by Arm leaves a delete. -func newBusWindow(ctx context.Context, api jetstream.JetStream, look func(batch string)) (*busWindow, error) { - kv, err := api.KeyValue(ctx, broker.MergeWindowBucket) - if err != nil { - return nil, fmt.Errorf("the merge window's bucket: %w", err) - } - w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) - if err != nil { - return nil, fmt.Errorf("watching the merge window: %w", err) - } - go func() { - defer w.Stop() - for { - select { - case <-ctx.Done(): - return - case e, ok := <-w.Updates(): - if !ok { - return - } - if e == nil || e.Operation() != jetstream.KeyValuePurge { - continue // a put, or a delete this controller made - } - if batch, _, found := strings.Cut(e.Key(), "."); found && look != nil { - look(batch) - } - } - } - }() - return &busWindow{kv: kv}, nil -} - -func (w *busWindow) Arm(ctx context.Context, batch string, quiet, most time.Duration) error { - // The most is made once, when the batch opens: it is there already for every merge after the first. - if _, err := w.kv.Create(ctx, batch+".most", []byte("most"), jetstream.KeyTTL(most)); err != nil && - !errors.Is(err, jetstream.ErrKeyExists) { - return err - } - // The quiet key is made again, not changed: a key's time to live is set when it is made. - if err := w.kv.Delete(ctx, batch+".quiet"); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - if _, err := w.kv.Create(ctx, batch+".quiet", []byte("quiet"), jetstream.KeyTTL(quiet)); err != nil { - return err - } - return nil -} - -func (w *busWindow) Forget(ctx context.Context, batch string) { - for _, key := range []string{batch + ".quiet", batch + ".most"} { - _ = w.kv.Delete(ctx, key) - } -} - -// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the -// next test replaces what they read. -var lookingAtBatches sync.WaitGroup - -// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches -// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once. -func lookAtBatches(ctx context.Context, open *stores) { - lookingAtBatches.Add(1) - defer lookingAtBatches.Done() - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err) - return - } - defer release() - if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil { - fmt.Printf("batches: %v\n", err) - } -} - -// startMergeWindow puts the merge window's clock on the bus for as long as the context lives: the controller is -// the lease holder here, the only one that writes it. The watcher looks at the batches when a key expires; the -// batches open now get their keys, and one look is taken, so a window that closed while no controller served is -// cut (ADR 0407 decision 1). No timer polls them. -func startMergeWindow(ctx context.Context, open *stores, js *broker.JetStream) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - look := func(string) { lookAtBatches(ctx, open) } - windowLook.Store(&look) - clock, err := newBusWindow(ctx, api, look) - if err != nil { - windowLook.Store(nil) - return err - } - setWindowClock(clock) - go func() { - <-ctx.Done() - setWindowClock(nil) - windowLook.Store(nil) - }() - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - return err - } - defer release() - batches, err := open.inventory.Batches(ctx) - if err != nil { - return err - } - now := time.Now().UTC() - for _, b := range batches { - armWindow(ctx, b, now) - } - return cutBatchesHeld(ctx, open, now) -} diff --git a/cmd/mesh-controller/merge_window_bus_integration_test.go b/cmd/mesh-controller/merge_window_bus_integration_test.go deleted file mode 100644 index 5cdae1e8..00000000 --- a/cmd/mesh-controller/merge_window_bus_integration_test.go +++ /dev/null @@ -1,90 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" -) - -func windowOnABus(t *testing.T) (*busWindow, <-chan string) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan string, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch }) - if err != nil { - t.Fatal(err) - } - return w, heard -} - -func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" { - t.Fatalf("heard %q (%v); want plan-1", b, ok) - } -} - -func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) { - w, heard := windowOnABus(t) - ctx := t.Context() - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was - t.Fatal(err) - } - if b, ok := hears(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("heard %q before the restarted window was up", b) - } - // Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s. - start := time.Now() - for time.Since(start) < 2500*time.Millisecond { - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(700 * time.Millisecond) - } - if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" { - t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok) - } -} - -func TestForgettingABatchWakesNobody(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - w.Forget(t.Context(), "plan-3") - if b, ok := hears(t, heard, 3*time.Second); ok { - t.Fatalf("a forgotten batch was heard expire: %q", b) - } -} diff --git a/cmd/mesh-controller/merge_window_bus_test.go b/cmd/mesh-controller/merge_window_bus_test.go deleted file mode 100644 index 760a360e..00000000 --- a/cmd/mesh-controller/merge_window_bus_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// windowOnABus is the merge window's clock on a real bus, and the batches its watcher heard expire. - -func hears(t *testing.T, heard <-chan string, within time.Duration) (string, bool) { - t.Helper() - select { - case b := <-heard: - return b, true - case <-time.After(within): - return "", false - } -} - -// A batch's quiet key expires when no merge came for the window, and the watcher names the batch (issue 973, -// novox/hq ADR 0407 decision 1). - -// A merge into the batch makes the quiet key again, so the window restarts: it does not expire at the first -// deadline, and the batch's most closes it all the same. - -// A batch cut is forgotten: its keys are taken away by a delete, which is no expiry and wakes nobody. - -// The deadlines a batch's keys are made with: what is left of its window and of its most, at least a second; -// none for a batch whose window is closed or that is no batch. -func TestTheDeadlinesOfABatchsKeys(t *testing.T) { - now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - batch := func(closes, most time.Duration, closed bool) inventory.Plan { - return inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(closes), AtMost: now.Add(most), Closed: closed}}} - } - for _, c := range []struct { - what string - b inventory.Plan - quiet, most time.Duration - armed bool - }{ - {"open", batch(90*time.Second, 10*time.Minute, false), 90 * time.Second, 10 * time.Minute, true}, - {"almost up", batch(100*time.Millisecond, 5*time.Second, false), time.Second, 5 * time.Second, true}, - {"closed by a stop", batch(90*time.Second, 10*time.Minute, true), 0, 0, false}, - {"up", batch(-time.Second, 10*time.Minute, false), 0, 0, false}, - {"cut at once", func() inventory.Plan { - b := batch(90*time.Second, 10*time.Minute, false) - b.Delivery.Batch.CutAtOnce = true - return b - }(), 0, 0, false}, - {"no batch", inventory.Plan{}, 0, 0, false}, - } { - quiet, most, armed := windowDeadlines(c.b, now) - if armed != c.armed || quiet != c.quiet || most != c.most { - t.Errorf("%s: %v, %v, %v; want %v, %v, %v", c.what, quiet, most, armed, c.quiet, c.most, c.armed) - } - } -} - -// A deadline that is not a whole number of seconds is rounded up, never down: the server rounds a key's time to live -// down to whole seconds, so 89.999999 s would expire at 89 s, before the window is closed by the record's times, and -// the look would cut nothing (review of issue 973). -func TestAFractionalDeadlineIsRoundedUpToTheNextWholeSecond(t *testing.T) { - now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - b := inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(89*time.Second + 999999*time.Microsecond), AtMost: now.Add(599*time.Second + time.Nanosecond)}}} - quiet, most, armed := windowDeadlines(b, now) - if !armed || quiet != 90*time.Second || most != 600*time.Second { - t.Errorf("deadlines %v, %v (armed %v); want 1m30s, 10m0s", quiet, most, armed) - } -} diff --git a/cmd/mesh-controller/merged_by_claim_test.go b/cmd/mesh-controller/merged_by_claim_test.go deleted file mode 100644 index 1b3544c7..00000000 --- a/cmd/mesh-controller/merged_by_claim_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "context" - "errors" - "testing" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// 1211/AC1 to AC3: the controller keeps a merge announced by a module that claims the delivery seat, whatever its -// name, and takes no word from one that does not (novox/hq ADR 0437 §3). With no bus held, a word that passes the -// claim is held for the bus (ErrTryAgain); a word that does not is refused before the bus is asked. -func TestOnlyAModuleClaimingTheSeatIsTheDeliverysWordOnAMerge(t *testing.T) { - entries := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("gitea"), namedOnly("mesh-delivery")} - keeper := keepingDeliveryMerges{ - js: func() *broker.JetStream { return nil }, - catalog: func(context.Context) ([]inventory.Entry, error) { return entries, nil }, - } - body := []byte(`{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve"}`) - for emitter, passes := range map[string]bool{ - "mesh-delivery-gate": true, // the claimant, under another name - "gitea": false, // claims no seat - "mesh-delivery": false, // named so, claiming no seat here - } { - err := keeper.KeepDeliveryMerge(context.Background(), emitter, body) - if passes != errors.Is(err, link.ErrTryAgain) || (!passes && !errors.Is(err, link.ErrNotTheDeliverysWord)) { - t.Errorf("%s: %v (passes the claim: %v)", emitter, err, passes) - } - } - entries = []inventory.Entry{claimingTheSeat("mesh-delivery")} - if err := keeper.KeepDeliveryMerge(context.Background(), "mesh-delivery", body); !errors.Is(err, link.ErrTryAgain) { - t.Errorf("today's module, claiming the seat: %v", err) - } -} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index f021b1b8..db18ca72 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -631,97 +631,6 @@ func accountOf(r inventory.ResourceHealth) string { return r.Target } -// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236 -// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since -// the send; not yet otherwise, saying which. A machine that never stated health is judged as before. -// -// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module -// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build, -// and not something a machine can meet within a bound. -func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) { - if f.healthErr != nil { - return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error()) - } - h, states := f.health[machine] - if !states { - return healthGood, "" - } - if h.HeardAt.Before(since) { - return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module) - } - // **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy. - resources := checkWaiting(machine, h.Resources, f.waits) - wait, waits := personWait(module, machine, resources) - // **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from - // what the controller sent, never from when the machine says the wait began — that time is the engine's - // memory, reset by its restart and moved by a change of words. A build that adds no account group cannot - // have brought a wait, so a later build sent while the login is still owed is judged as before. - if waits && !f.groupsAdded[module] { - waits = false - } - var found, onWaiting []string - var forOperator []inventory.Wait - for _, r := range resources { - if r.Module != module { - continue - } - // **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not - // given is owed by every build alike, so it is no fault of this one, and the verdict carries it. - if r.State == link.StateWaiting { - forOperator = append(forOperator, r.Waits...) - continue - } - if waits && r.State == link.StateUnhealthy { - continue - } - // **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine - // left its owner and mode, and only someone at the machine can hand it over. It is no fault of this - // build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict - // carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy. - if usedAsFound(r) && r.Since.Before(since) { - found = append(found, r.Resource) - continue - } - switch r.State { - case link.StateHealthy: - case link.StateStarting: - return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine) - case link.StateUnhealthy: - if on, held := f.heldOn[module+"@"+machine]; held { - // **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a - // person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the - // operator's secret, whichever module owes it. - if on.waits != "" { - onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s", - r.Kind, r.Resource, machine, on.on, on.waits)) - continue - } - return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind, - r.Resource, machine, on.on) - } - return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r)) - default: - return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State, - reasonAfter(r.Reason)) - } - } - if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 { - said := onWaiting - if waits { - said = append(said, wait) - } - if len(forOperator) > 0 { - said = append(said, operatorWaitSaid(module, machine, forOperator)) - } - if len(found) > 0 { - said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+ - "(`nox node hand-over %s ` on the control-node)", machine, module, strings.Join(found, ", "), machine)) - } - return healthPerson, strings.Join(said, "; ") - } - return healthGood, "" -} - // usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339). func usedAsFound(r inventory.ResourceHealth) bool { return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) diff --git a/cmd/mesh-controller/modules_wire_store_test.go b/cmd/mesh-controller/modules_wire_store_test.go index ea3fcd6f..2e3dfb7d 100644 --- a/cmd/mesh-controller/modules_wire_store_test.go +++ b/cmd/mesh-controller/modules_wire_store_test.go @@ -2,82 +2,4 @@ package main -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/link" -) - -// 1145/AC1 on a store: declarationWith itself sets the module of each resource and the build of each module for a -// node recorded as reading them, and for no other. Skips without a store. -func TestAComposedDeclarationCarriesModulesAndBuildsOnlyForANodeThatReadsThem(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - manifest := resticFixture(false) - manifest["module"], manifest["version"] = "restic", "1" - raw, _ := json.Marshal(manifest) - b := link.BuildResult{ID: link.NewBuildID(start), Repository: "novox/mesh-catalog", Path: "modules/restic", - On: "anchor", Module: "restic", Commit: "c1", Manifest: raw, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "restic"); err != nil { - t.Fatal(err) - } - } - anchor, err := inv.NodeByName(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordReadsModules(ctx, anchor.ID, true); err != nil { - t.Fatal(err) - } - wantBuild, err := inv.BuildOf(ctx, "restic", "c1") - if err != nil || wantBuild == "" { - t.Fatalf("the build of restic at c1 is %q (%v)", wantBuild, err) - } - - declare := func(node string) sendable { - t.Helper() - d, err := composeSend(ctx, open, node) - if err != nil { - t.Fatal(err) - } - return d - } - reads := bodyKeys(t, declare("anchor")) - builds, _ := reads["builds"].(map[string]any) - if builds["restic"] != wantBuild { - t.Errorf("1145/AC1: anchor reads modules and was sent builds %v, want restic at %s", reads["builds"], wantBuild) - } - owned := 0 - for _, r := range reads["resources"].([]any) { - r := r.(map[string]any) - if strings.HasPrefix(r["id"].(string), "restic.") { - owned++ - if r["module"] != "restic" { - t.Errorf("1145/AC1: resource %v carries module %v, want restic", r["id"], r["module"]) - } - } - } - if owned == 0 { - t.Fatal("1145/AC1: no resource of restic was composed") - } - - plain := bodyKeys(t, declare("laptop")) - if _, there := plain["builds"]; there { - t.Errorf("1145/AC1: laptop never said it reads modules and was sent builds %v", plain["builds"]) - } - for _, r := range plain["resources"].([]any) { - if _, there := r.(map[string]any)["module"]; there { - t.Fatalf("1145/AC1: laptop was sent a module on %v", r) - } - } -} +import () diff --git a/cmd/mesh-controller/modules_wire_test.go b/cmd/mesh-controller/modules_wire_test.go index 7dd25721..344a7bf1 100644 --- a/cmd/mesh-controller/modules_wire_test.go +++ b/cmd/mesh-controller/modules_wire_test.go @@ -24,22 +24,6 @@ func modulesDeclaration(reads bool) sendable { } } -// 1145/AC1 -func TestAMachineThatReadsModulesIsSentEachResourcesModuleAndEachModulesBuild(t *testing.T) { - keys := bodyKeys(t, modulesDeclaration(true)) - builds, _ := keys["builds"].(map[string]any) - if len(builds) != 1 || builds["web.app"] != "build-7" { - t.Fatalf("1145/AC1: the body carries builds %v", keys["builds"]) - } - want := map[string]any{"web.app.conf": "web.app", "web.app.data": "web.app", "mesh.guard": nil} - for _, r := range keys["resources"].([]any) { - r := r.(map[string]any) - if r["module"] != want[r["id"].(string)] { - t.Errorf("1145/AC1: resource %v carries module %v, want %v", r["id"], r["module"], want[r["id"].(string)]) - } - } -} - // 1145/AC2: the body is byte for byte what it is today. func TestAMachineThatDoesNotReadModulesIsSentTheBodyItWasSentBefore(t *testing.T) { d := modulesDeclaration(false) diff --git a/cmd/mesh-controller/moved.go b/cmd/mesh-controller/moved.go index 6c07c210..d9c57391 100644 --- a/cmd/mesh-controller/moved.go +++ b/cmd/mesh-controller/moved.go @@ -284,14 +284,14 @@ func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, as if !onForge { return "", errors.New("the repository is not on the mesh's forge") } - merged, known, err := inv.MergeOf(ctx, owner+"/"+name, commit) + merged, known, err := keptForgeMerge(ctx, owner, name, commit) if err != nil { return "", err } if !known { return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, commit) } - return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged)) + return approvedAnnounced(ctx, inv, owner, name, merged) } // errWordNotAnApprove says the merge cannot be shown to be the operator's Approve of its head: its announcement names no @@ -330,3 +330,81 @@ func approvedAnnounced(ctx context.Context, inv catalogueReader, owner, name str } return fmt.Sprintf("%s/%s#%d merged at %.8s on %s", owner, name, k.Number, k.HeadSHA, why), nil } + +// claimantsOf are the modules among the catalogued entries that claim a seat, each once, in name order. +func claimantsOf(entries []inventory.Entry, seat string) []string { + var out []string + for _, e := range entries { + if e.Manifest.ClaimsSeat(seat) && !slices.Contains(out, e.Manifest.Module) { + out = append(out, e.Manifest.Module) + } + } + sort.Strings(out) + return out +} + +// forgeMergeKey is where the forge's announcement of a merge is kept, by the merge commit: the one thing a build of a merge +// names, from which the pull request and its head are found (novox/hq ADR 0420 §7). +func forgeMergeKey(owner, repo, commit string) string { + return fmt.Sprintf("merge/%s/%s/%s", owner, repo, commit) +} + +// keepForgeMerge keeps the forge's announcement of a merge in the bucket of the merges, a week at most as that bucket +// keeps them. The controller plans nothing from a merge any more (ADR 0539 §17); this is all it keeps of one. +var keepForgeMerge = func(ctx context.Context, m link.SourceMoved) error { + serving := servingBus.Load() + if serving == nil { + return fmt.Errorf("%w: the bus is not held", link.ErrTryAgain) + } + api, err := jetstream.New(serving.Conn()) + if err != nil { + return fmt.Errorf("%w: %w", link.ErrTryAgain, err) + } + kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket) + if err != nil { + return fmt.Errorf("%w: the bucket of the merges: %w", link.ErrTryAgain, err) + } + body, err := json.Marshal(keptMerge{SourceMoved: m}) + if err != nil { + return err + } + if _, err := kv.Put(ctx, forgeMergeKey(m.Owner, m.Repo, m.Commit), body); err != nil { + return fmt.Errorf("%w: %w", link.ErrTryAgain, err) + } + return nil +} + +// keptForgeMerge is the forge's announcement of a merge, by its commit; false when none was kept. A variable so a test can +// hand it one. +var keptForgeMerge = func(ctx context.Context, owner, repo, commit string) (keptMerge, bool, error) { + serving := servingBus.Load() + if serving == nil { + return keptMerge{}, false, errors.New("only the serving controller keeps what the forge announced") + } + api, err := jetstream.New(serving.Conn()) + if err != nil { + return keptMerge{}, false, err + } + kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket) + if err != nil { + return keptMerge{}, false, err + } + e, err := kv.Get(ctx, forgeMergeKey(owner, repo, commit)) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return keptMerge{}, false, nil + } + if err != nil { + return keptMerge{}, false, err + } + var k keptMerge + if err := json.Unmarshal(e.Value(), &k); err != nil { + return keptMerge{}, false, err + } + return k, true, nil +} + +// keptMerge is a merge as the record keeps it: the forge's announcement. +type keptMerge struct { + link.SourceMoved + Moves []string `json:"moves,omitempty"` +} diff --git a/cmd/mesh-controller/moved_bus_test.go b/cmd/mesh-controller/moved_bus_test.go index 277295bd..26e99fb8 100644 --- a/cmd/mesh-controller/moved_bus_test.go +++ b/cmd/mesh-controller/moved_bus_test.go @@ -103,8 +103,11 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh const commit = "c0ffee0123456789" event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { + var forgeMerge link.SourceMoved + if err := json.Unmarshal(event, &forgeMerge); err != nil { + t.Fatal(err) + } + if err := keepForgeMerge(ctx, forgeMerge); err != nil { t.Fatal(err) } id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) diff --git a/cmd/mesh-controller/new_head_words_test.go b/cmd/mesh-controller/new_head_words_test.go deleted file mode 100644 index 86c8a959..00000000 --- a/cmd/mesh-controller/new_head_words_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// novox/hq ADR 0505 §5: a delivery whose cure is its author's new head reaches the operator in words that name its -// author and say a new head is needed, offer no Stop and never call it the operator's; the line as mesh-delivery-gate -// says it (ready.go readyLines: needs "new-head", author, branch). -func TestADeliveryThatNeedsANewHeadNamesItsAuthorAndOffersNoStop(t *testing.T) { - l := stalledLine{ID: "novox/mesh-controller@499ff5c21861", Number: 432, State: "rejected", For: "2h1m0s", Bound: "2h0m0s", - H2: "none: a new head from its author, or a recheck that passes", Needs: needsNewHead, - Author: "the author of feat/quiet-deliveries (quiet-deliveries on shanks)", Branch: "feat/quiet-deliveries", - Says: "novox/mesh-controller#432 passed its checks at 499ff5c21861 and is not merged: a conflict"} - o := stalledObservations([]stalledLine{l})[0] - for _, words := range []string{o.Headline, o.Explanation, o.Resolved, o.Needs} { - if why, ok := conditions.Plain(words); !ok { - t.Errorf("%q is not plain: %s", words, why) - } - for _, never := range []string{"stop it", "operator's"} { - if strings.Contains(words, never) { - t.Errorf("%q says %q", words, never) - } - } - } - if o.Headline != "Pull request 432 of mesh-controller needs a new head from its author" || - !strings.Contains(o.Explanation, "until the author of feat/quiet-deliveries (quiet-deliveries on shanks) pushes a new head") || - !strings.Contains(o.Explanation, "for 2 hours") || len(o.Actions) != 0 || - o.Needs != "ask the author of feat/quiet-deliveries (quiet-deliveries on shanks) for a new head." { - t.Fatalf("words: %q / %q / %q / %v", o.Headline, o.Explanation, o.Needs, o.Actions) - } - // A branch in code is not said; the author is then "its author". - l.Author = "the author of fix/a1b2c3d4e5f6" - if o := stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Needs, "ask its author") { - t.Fatalf("a dotted branch was said: %q", o.Needs) - } -} diff --git a/cmd/mesh-controller/new_module_approved_test.go b/cmd/mesh-controller/new_module_approved_test.go index 67725f52..1780a81b 100644 --- a/cmd/mesh-controller/new_module_approved_test.go +++ b/cmd/mesh-controller/new_module_approved_test.go @@ -6,7 +6,6 @@ import ( "errors" "strings" "testing" - "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -19,7 +18,7 @@ import ( type anEmptyMesh struct { request inventory.BuildRequest asked bool - merge inventory.BatchedMerge + merge keptMerge } func (m *anEmptyMesh) Catalogue(context.Context) (map[string]catalogue.Manifest, error) { @@ -33,9 +32,6 @@ func (m *anEmptyMesh) Catalogued(context.Context) ([]inventory.Entry, error) { r func (m *anEmptyMesh) BuildRequestByID(_ context.Context, id string) (inventory.BuildRequest, bool, error) { return m.request, m.asked && id == m.request.ID, nil } -func (m *anEmptyMesh) MergeOf(context.Context, string, string) (inventory.BatchedMerge, bool, error) { - return m.merge, true, nil -} // aDeliveryMergeOf is the Approved merge of a pull request of novox/mesh-delivery, its heads keyed by that repository, // and the ask and warrant bound to exactly those heads (the digest the Approve left covers them). @@ -102,8 +98,12 @@ func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwis } event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7, "head_sha": head, "merge_commit_sha": commit}) - mesh := &anEmptyMesh{asked: true, merge: inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", - Commit: commit, Merged: time.Now(), Event: event}, + var forgeMerge keptMerge + _ = json.Unmarshal(event, &forgeMerge) + wasForge := keptForgeMerge + t.Cleanup(func() { keptForgeMerge = wasForge }) + keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil } + mesh := &anEmptyMesh{asked: true, request: inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "", Ref: "main", Commit: commit, For: c.request}} forms := &sourceForms{bases: map[string]string{}, base: func(seat string) string { diff --git a/cmd/mesh-controller/new_module_bus_test.go b/cmd/mesh-controller/new_module_bus_test.go index 03ff6e9a..2300c0a3 100644 --- a/cmd/mesh-controller/new_module_bus_test.go +++ b/cmd/mesh-controller/new_module_bus_test.go @@ -104,8 +104,11 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB const commit = "c0ffee0123456789" event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7, "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { + var forgeMerge link.SourceMoved + if err := json.Unmarshal(event, &forgeMerge); err != nil { + t.Fatal(err) + } + if err := keepForgeMerge(ctx, forgeMerge); err != nil { t.Fatal(err) } id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) diff --git a/cmd/mesh-controller/new_module_on_merge_test.go b/cmd/mesh-controller/new_module_on_merge_test.go index 4a763321..73db2ae5 100644 --- a/cmd/mesh-controller/new_module_on_merge_test.go +++ b/cmd/mesh-controller/new_module_on_merge_test.go @@ -39,9 +39,6 @@ func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T) if err := (following{open: open}).SourceMoved(ctx, m); err != nil { t.Fatal(err) } - if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { - t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) - } // The indexer writes the module's trunk value; the builder builds it, then its key is done. fp := fpOf('7') manifest := b.indexed(t, "systemd-resolved", "1", "novox/mesh-catalog", "modules/systemd-resolved", commit, fp) diff --git a/cmd/mesh-controller/new_modules.go b/cmd/mesh-controller/new_modules.go index faee9615..0273ff3d 100644 --- a/cmd/mesh-controller/new_modules.go +++ b/cmd/mesh-controller/new_modules.go @@ -201,9 +201,16 @@ func newModuleObservations(now time.Time) []conditions.Observation { // looks at every module once now. A bucket not there yet is said: the look at start and a first merge's word still work. func watchNewModules(ctx context.Context, open *stores) { inBackground(func() { - answerHeldMerges(ctx, open.inventory) takeNewModules(ctx, open) }) + // And, beside it, the start's pass of release 6 and the take-in of each registered module's new done key (novox/hq + // ADR 0539 clauses 5, 7, 10 and 13; composed_start.go, composed_keys.go): the plans' buckets deleted, every node + // composed, the replaced controller composed, and from now on each new fingerprint registered, composed and granted. + inBackground(func() { composeAtStart(ctx, open) }) + watchDoneKeys(ctx, open) + watchHavesForJudgement(ctx, open) + // And the keep set keeps the builds every want names (ADR 0539 clause 1), with no plan left to name them. + open.inventory.SetWanted(wantedOnTheBus) keys, closeKeys, err := openBuilderKeys() if err != nil { fmt.Printf("new modules: the builder's keys are not watched: %v\n", err) @@ -232,8 +239,6 @@ func wakeNewModule(ctx context.Context, open *stores, module string) { } for _, e := range entries { if e.Manifest.Module == module { - // Known: a plan waiting on its key is woken by the plans' own look. - askToLook() return } } diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index 232bd89e..f1ae066f 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -616,10 +616,11 @@ func throughTheTunnel(ctx context.Context, open *stores, node inventory.Node, ke } } } - if err := sendTo(ctx, open, []string{hub.Name}); err != nil { - return nil, "", fmt.Errorf("%s was made a peer of the hub, and the hub could not be sent "+ - "it, so the tunnel would not answer — the token is not shown; issue it again once %s "+ - "can be pushed: %w", node.Name, hub.Name, err) + // The hub's own composition carries the private network (novox/hq ADR 0539 clause 5), its want then names it. + if _, err := composeModules(ctx, open, []string{hub.Name}); err != nil { + return nil, "", fmt.Errorf("%s was made a peer of the hub, and the hub's composition could not be "+ + "written, so the tunnel would not answer — the token is not shown; issue it again once %s "+ + "composes: %w", node.Name, hub.Name, err) } // An address, not a name — nothing resolves before the machine has joined (novox/hq ADR 0004). return &token.Tunnel{ @@ -796,19 +797,6 @@ func showNode(ctx context.Context, stored *stores, name string) error { if err := showMode(ctx, inv, node); err != nil { return err } - // What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what - // it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader. - // Unreadable is said, not taken for none. - switch last, found, err := inv.LastSend(ctx, name); { - case err != nil: - fmt.Printf(" what it was last sent could NOT be read: %v\n", err) - case !found: - fmt.Printf(" last sent no send recorded since the mesh began keeping them\n") - default: - for _, line := range lastSendLines(last) { - fmt.Println(line) - } - } // Whom agents run as here, and whether that account can become root without a person (ADR 0266). for _, line := range agentAccountLines(ctx, inv, node) { fmt.Println(line) @@ -915,12 +903,6 @@ func showNode(ctx context.Context, stored *stores, name string) error { } if len(assigned) > 0 { fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) - // And what each runs, as its assignment names it (novox/hq ADR 0408, issue 1095). - if on, err := inv.AssignmentsOn(ctx, name); err == nil { - for _, line := range runLines(on) { - fmt.Println(line) - } - } } // And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same // reason its condition says. Not computable is said, never read as "all applied". diff --git a/cmd/mesh-controller/non_trunk_merge_test.go b/cmd/mesh-controller/non_trunk_merge_test.go deleted file mode 100644 index 71322750..00000000 --- a/cmd/mesh-controller/non_trunk_merge_test.go +++ /dev/null @@ -1,55 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A merge into a branch other than the trunk starts no delivery (novox/hq ADR 0401 clause 3, issue 889): a -// module recorded with no ref follows the trunk, main, and no other branch. -func TestAMergeIntoANonTrunkBranchStartsNoDelivery(t *testing.T) { - for ref, name := range map[string]string{"": "noref", "main": "mainref"} { - t.Run(name, func(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "free", Version: "1"}, - inventory.Source{Repository: "novox/free", Seat: "git", Ref: ref, BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - feature := lone(repoMerge("free", "f1", t0)) - feature.Base = "feat/57-config-and-provisioning" - said := captured(t, func() error { return (following{open: open}).hearMerge(t.Context(), feature, t0) }) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 0 { - t.Fatalf("a merge into a feature branch: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - if !strings.Contains(said, feature.Base) || !strings.Contains(said, "starts no delivery") { - t.Fatalf("the merge was not said to start no delivery: %q", said) - } - hear(t, open, lone(repoMerge("free", "m1", t0)), t0) - if ws, bs := walks(t, open); len(ws)+len(bs) != 1 { - t.Fatalf("a merge into main: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - }) - } -} - -// A module recorded with an explicit branch keeps following that branch (the lead's settlement of issue 977 Q5). -func TestAModuleRecordedWithABranchStillFollowsIt(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "pinned", Version: "1"}, - inventory.Source{Repository: "novox/pinned", Seat: "git", Ref: "feat/x", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := lone(repoMerge("pinned", "p1", t0)) - m.Base = "feat/x" - hear(t, open, m, t0) - if ws, bs := walks(t, open); len(ws)+len(bs) != 1 { - t.Fatalf("a merge into the branch a module follows: %d walk(s), %d batch(es)", len(ws), len(bs)) - } -} diff --git a/cmd/mesh-controller/one_group_one_plan_test.go b/cmd/mesh-controller/one_group_one_plan_test.go deleted file mode 100644 index f37d07b0..00000000 --- a/cmd/mesh-controller/one_group_one_plan_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// Acceptance 1 of design 47, the controller's part: a feature across the controller and the catalogue with a new seat -// verb is delivered as one group, with no `after:` line — its merges are one batch, each module is built once and sent -// once, its commits are carried by that one plan, and the plan names the edge that placed each tier. The Approve and -// the one-pass merge are mesh-delivery's. - -// manifestsHeld stubs the forge for a test: the manifest each (repository, path, commit) holds, and none anywhere else. -func manifestsHeld(t *testing.T, held map[[3]string]catalogue.Manifest) { - t.Helper() - was, wasParent := manifestAt, parentOf - t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) - manifestsReadMu.Lock() - clear(manifestsRead) - manifestsReadMu.Unlock() - parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } - manifestAt = func(_ context.Context, _, repo, file, ref string) (catalogue.Manifest, bool, error) { - m, ok := held[[3]string{repo, file, ref}] - return m, ok, nil - } -} - -func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *testing.T) { - for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} { - t.Run(name, func(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - ctx := t.Context() - const head = "feat/one-group-one-plan" - // mesh-delivery holds its seat, so a catalogue plan would wait for its word; this plan carries the controller. - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - controller := fromHead(repoMerge("mesh-controller", "k1", t0), head) - controller.Paths = []string{"module.json", "cmd/mesh-controller/main.go"} - catalog := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), head) - manifestsHeld(t, map[[3]string]catalogue.Manifest{ - {"mesh-controller", "module.json", "k1"}: {Module: "mesh-controller", - DefinesSeats: []catalogue.SeatDeclaration{{Name: "seat", Serves: []catalogue.Verb{{Name: "verb"}}}}}, - {"mesh-catalog", "modules/app/module.json", "cat1aaaaaaaa"}: {Module: "app", Invokes: []string{"seat:seat.verb"}}, - }) - states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} - groupStates(t, head, states) - first, second := controller, catalog - if !controllerFirst { - first, second = catalog, controller - } - // The first member merges; the group waits for the second, however long it takes. - hear(t, open, first, t0) - cutAt(t, open, t0.Add(30*time.Minute)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { - t.Fatalf("a member of a group alone: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - states["novox/"+second.Repo] = "merged" - second.MergedAt = t0.Add(30 * time.Minute).Format(time.RFC3339Nano) - hear(t, open, second, t0.Add(30*time.Minute)) - cutAt(t, open, t0.Add(30*time.Minute+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("after the window: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs)) - } - plan := ws[0] - if plan.CommitOf("novox/mesh-controller") != "k1" || plan.CommitOf("novox/mesh-catalog") != "cat1aaaaaaaa" { - t.Fatalf("the plan carries %+v", plan.Commits) - } - if len(plan.Tiers) != 2 || strings.Join(plan.Tiers[0], ",") != "mesh-controller" || strings.Join(plan.Tiers[1], ",") != "app" { - t.Fatalf("the plan's tiers are %v; want the controller, then the caller of its verb", plan.Tiers) - } - if placed := plan.Modules["app"].Placed; !strings.Contains(placed, "verb before caller: seat.verb") || - !strings.Contains(placed, "after mesh-controller") { - t.Fatalf("the plan does not name the edge that placed the caller: %q", placed) - } - if plan.Modules["mesh-controller"].Placed != "" { - t.Fatalf("the first tier is placed by %q", plan.Modules["mesh-controller"].Placed) - } - if plan.Delivery != nil && (plan.Delivery.Awaits != "" || plan.Waiting()) { - t.Fatalf("a plan carrying the controller waits for %q's word", plan.Delivery.Awaits) - } - // The cut takes the first tier from the controller's key at its commit and asks no build; the take-in itself is a - // later look's, which this test does not run, so nothing is recorded as asked. - if c := plan.Modules["mesh-controller"]; !c.Key || c.State != "asked" || c.KeyAt != "k1" || len(*asked) != 0 { - t.Fatalf("the first tier is not taken from its key once: %+v, %d ask(s) %v", c, len(*asked), *asked) - } - // Every merge is answered by that one plan. - for _, repository := range []string{"novox/mesh-controller", "novox/mesh-catalog"} { - merges, err := open.inventory.MergesOf(ctx, plan.ID) - if err != nil || len(merges) != 2 { - t.Fatalf("the plan answers %d merge(s) of %s: %v", len(merges), repository, err) - } - } - }) - } -} - -// Issue 1049: a move delivered half before its second merge. The controller adds a module the catalogue removes, in one -// group: the merges are one batch and one plan in either order, no build of the module is asked (it is registered from -// the builder's key, ADR 0501 §6), and the catalogue's removal does not forget it. -func TestAMoveOfAModuleAcrossTwoRepositoriesIsOnePlanAndBuildsItOnceFromTheNewOne(t *testing.T) { - for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} { - t.Run(name, func(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - ctx := t.Context() - const head = "feat/whole-modules" - // The module runs on a machine from the catalogue's repository, so its removal there cannot forget it. - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "route-proxy", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/route-proxy", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "route-proxy"); err != nil { - t.Fatal(err) - } - manifestsHeld(t, nil) - controller := fromHead(repoMerge("mesh-controller", "k1", t0), head) - controller.Paths = []string{"modules/route-proxy/module.json", "modules/route-proxy/main.go"} - controller.ModuleDirs, controller.ModuleDirsSaid = []string{"modules/route-proxy"}, true - catalog := fromHead(catalogueMerge("cat1aaaaaaaa", "route-proxy", t0), head) - catalog.Removed = []string{"modules/route-proxy/module.json"} - catalog.ModuleDirs, catalog.ModuleDirsSaid = nil, true - states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} - groupStates(t, head, states) - first, second := controller, catalog - if !controllerFirst { - first, second = catalog, controller - } - hear(t, open, first, t0) - states["novox/"+second.Repo] = "merged" - second.MergedAt = t0.Add(20 * time.Second).Format(time.RFC3339Nano) - hear(t, open, second, t0.Add(20*time.Second)) - cutAt(t, open, t0.Add(20*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("a move: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs)) - } - // **No build of it is asked from either repository** (ADR 0501 §6, issue 1851): the move is registered from the - // builder's done key of route-proxy in the controller's repository, on the operator's Approve of the group. - for _, a := range *asked { - if a[1] == "modules/route-proxy" { - t.Errorf("a build of route-proxy was asked: %v", a) - } - } - entries, err := open.inventory.Catalogued(ctx) - if err != nil { - t.Fatal(err) - } - kept := false - for _, e := range entries { - kept = kept || e.Manifest.Module == "route-proxy" - } - if !kept { - t.Fatal("the catalogue's removal forgot route-proxy, which a machine runs") - } - }) - } -} diff --git a/cmd/mesh-controller/operator_wait_integration_test.go b/cmd/mesh-controller/operator_wait_integration_test.go deleted file mode 100644 index 98a468ef..00000000 --- a/cmd/mesh-controller/operator_wait_integration_test.go +++ /dev/null @@ -1,133 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("raised on one statement") - } - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil { - t.Fatal(err) - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("not raised on two statements: %+v", open) - } - for _, c := range open { - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault too: %+v", c) - } - } - if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning { - t.Fatalf("the condition: %+v", got) - } - if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") || - !strings.Contains(got.Explanation, "hidden prompt opens at the desk") { - t.Fatalf("its needs do not name the act: %q", got.Needs) - } - if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") { - t.Fatalf("its explanation: %q", got.Explanation) - } - if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") { - t.Fatalf("its summary does not name the secret and the line: %q", got.Summary) - } - // Long open is still a warning: only the operator can end it. - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent { - t.Fatalf("after two days: %+v", got) - } - // Given: the next statement does not say it, and it clears. - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("not cleared once given") - } -} - -func TestASettingsWaitAsksForTheApproval(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, _ := needsOperatorOpen(t, k) - if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") { - t.Fatalf("the condition: %+v", got) - } -} - -func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, - factsGiven(map[string]time.Time{"smb-password-games": time.Now()})) - rs := map[string][]inventory.ResourceHealth{"mounts": checked} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatalf("a wait for a secret given raised needs-operator: %+v", got) - } - unhealthy := false - for _, c := range open { - unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !unhealthy { - t.Fatalf("not raised as unhealthy: %+v", open) - } -} - -func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - broken := waitingResource() - broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil - for i := 3; i <= 4; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}}, - map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatal("needs-operator still open after it became a fault") - } - found := false - for _, c := range open { - found = found || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !found { - t.Fatalf("the fault is not raised: %+v", open) - } -} diff --git a/cmd/mesh-controller/operator_wait_test.go b/cmd/mesh-controller/operator_wait_test.go deleted file mode 100644 index 15879fa7..00000000 --- a/cmd/mesh-controller/operator_wait_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386). - -var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1", - Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}}, - OwnSecrets: catalogue.OwnSecrets{ - "smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside}, - "broker": {Path: "/s/broker"}, - "made": {Path: "/s/made", Taken: catalogue.TakenAtStart}, - "licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside}, - }} - -var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games", - What: "the password of the source games"} - -var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"} - -func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process", - Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool", - Reason: "the source games waits for its password", Waits: waits} -} - -func factsGiven(given map[string]time.Time) operatorWaitFacts { - return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}, - given: map[string]map[string]time.Time{"mounts@workstation": given}} -} - -// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there. -func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) { - at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC) - for _, c := range []struct { - name string - w inventory.Wait - f operatorWaitFacts - says string // "" when excused - }{ - {"a member of an outside family, not given", passwordWait, factsGiven(nil), ""}, - {"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""}, - {"a declared setting", usernameWait, factsGiven(nil), ""}, - {"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"}, - {"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"}, - {"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"}, - {"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"}, - {"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"}, - {"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"}, - {"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"}, - {"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"}, - {"what was given cannot be read", passwordWait, - operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"}, - } { - err := checkWait("mounts", "workstation", c.w, c.f) - switch { - case c.says == "" && err != nil: - t.Errorf("%s: refused: %v", c.name, err) - case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)): - t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says) - } - } -} - -// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why. -func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) { - given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()}) - got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given) - if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") { - t.Fatalf("a wait for a secret given: %+v", got[0]) - } - got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil)) - if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") { - t.Fatalf("a wait naming nothing: %+v", got[0]) - } - got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil)) - if got[0].State != link.StateWaiting { - t.Fatalf("two waits that check out: %+v", got[0]) - } -} - -// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything -// else beside it is judged as before; and any build is excused, not only one that added something. -func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) { - now := time.Now() - since := now.Add(-time.Minute) - healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process", - Target: "mesh-mounts-apply.service", State: link.StateHealthy} - f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false}, - health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now, - Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}} - h, why := moduleHealthWord("mounts", "workstation", since, f) - if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") || - !strings.Contains(why, "nox secret ask workstation mounts smb-password-games") { - t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why) - } - // A second resource unhealthy beside it: not yet, as before. - down := healthy - down.State, down.Reason = link.StateUnhealthy, "down" - f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now, - Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}} - if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet { - t.Fatalf("a resource down beside the wait reads %v %q", h, why) - } - // The password given and the module still saying it waits: not excused. - f.waits = factsGiven(map[string]time.Time{"smb-password-games": now}) - f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now, - Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}} - if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") { - t.Fatalf("a wait for a secret given reads %v %q", h, why) - } - // Facts never read (no manifest): never excused. - f.waits = operatorWaitFacts{} - if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet { - t.Fatalf("a wait nothing could check reads %v", h) - } -} - -func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) { - t.Helper() - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - for i, c := range open { - if c.Key == needsOperatorKey("mounts", "workstation") { - return &open[i], open - } - } - return nil, open -} - -// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the -// act; a statement without it clears it. - -// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator. - -// A module that waited and is then broken says so when the wait clears, and the other way round. diff --git a/cmd/mesh-controller/order_test.go b/cmd/mesh-controller/order_test.go index 5aa455b1..9163e23f 100644 --- a/cmd/mesh-controller/order_test.go +++ b/cmd/mesh-controller/order_test.go @@ -75,52 +75,6 @@ func fromRepo(module, repository, path string) inventory.Entry { } } -// A merge rebuilds the modules whose own directories it changed, and everything when what it changed -// is shared. One repository holding many modules is the ordinary case here, and rebuilding all of -// them for a change to one is what exhausted a registry's pull limit the first night this ran. -func TestAMergeRebuildsTheModulesItChanged(t *testing.T) { - const repo = "http://forge.internal:20000/novox/mesh-catalog.git" - gitea := fromRepo("gitea", repo, "modules/gitea") - keycloak := fromRepo("keycloak", repo, "modules/keycloak") - known := []inventory.Entry{gitea, keycloak, fromRepo("plex", repo, "modules/plex")} - candidates := []inventory.Entry{gitea, keycloak} - merge := func(paths []string, truncated bool) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", - Paths: paths, PathsTruncated: truncated} - } - named := func(entries []inventory.Entry) string { - var names []string - for _, e := range entries { - names = append(names, e.Manifest.Module) - } - return strings.Join(names, ",") - } - for _, c := range []struct { - what string - m link.SourceMoved - want string - }{ - {"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"}, - {"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"}, - {"a file at the root no build reads (issue 280)", merge([]string{"tsconfig.json"}, false), ""}, - {"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""}, - {"nothing said about the files", merge(nil, false), "gitea,keycloak"}, - {"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"}, - // novox/hq issue 252: a module the mesh has never registered is still a module, when the merge - // shows it is one — and a directory that may be shared code is still shared. - {"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"}, - {"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""}, - {"a module removed", merge([]string{"modules/gone/module.json"}, false), ""}, - {"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), ""}, - {"a file directly among the modules", merge([]string{"modules/README.md"}, false), ""}, - {"a root file beside a module's", merge([]string{"merge-check.sh", "modules/keycloak/x.ts"}, false), "keycloak"}, - } { - if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want { - t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want) - } - } -} - // A module whose recipe packages source from another repository is affected when that repository // moves — the manifest the mesh keeps says nothing about it, so the record of what the build read is // the only thing that can say so. @@ -203,52 +157,6 @@ func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) { } } -// novox/hq issues 278 and 280: a merge touching the code of a module the mesh does not hold — the -// catalogue's reference module, whose manifest it left alone — read as shared and rebuilt every module -// built from the repository (103 of them on 2026-10-06, 88 byte-identical); so did a merge-check.sh added at -// the root. No build reads a file outside its module's directory, so neither rebuilds anything, said by the -// announcer or not. -func TestAChangeInsideAModuleIsThatModulesHeldOrNot(t *testing.T) { - const repo = "http://forge.internal:20000/novox/mesh-catalog.git" - gitea := fromRepo("gitea", repo, "modules/gitea") - keycloak := fromRepo("keycloak", repo, "modules/keycloak") - known := []inventory.Entry{gitea, keycloak} - candidates := []inventory.Entry{gitea, keycloak} - merge := func(paths, modules []string, said bool) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Paths: paths, - ModuleDirs: modules, ModuleDirsSaid: said} - } - named := func(entries []inventory.Entry) string { - var names []string - for _, e := range entries { - names = append(names, e.Manifest.Module) - } - return strings.Join(names, ",") - } - showcase := []string{"modules/showcase/index.ts"} - for _, c := range []struct { - what string - m link.SourceMoved - want string - }{ - {"a module held by none, said", merge(showcase, []string{"modules/showcase"}, true), ""}, - {"beside a held one's change", merge(append([]string{"modules/gitea/index.ts"}, showcase...), - []string{"modules/gitea", "modules/showcase"}, true), "gitea"}, - {"deeper inside it", merge([]string{"modules/showcase/daemon/index.ts"}, []string{"modules/showcase"}, true), ""}, - {"a directory holding no manifest, read by no build", merge([]string{"modules/lib/x.go"}, nil, true), ""}, - {"one of two files in no module", merge([]string{"modules/showcase/index.ts", "modules/lib/x.go"}, - []string{"modules/showcase"}, true), ""}, - {"the root is never a module directory", merge([]string{"tsconfig.json"}, []string{"", "/", "."}, true), ""}, - {"not said: still no build reads it", merge(showcase, []string{"modules/showcase"}, false), ""}, - {"an old announcer saying nothing", merge(showcase, nil, false), ""}, - {"a manifest the merge removed, said or not", merge([]string{"modules/gone/module.json", "modules/gone/x.ts"}, nil, true), ""}, - } { - if got := named(whatTheMergeTouched(candidates, known, c.m, nil)); got != c.want { - t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want) - } - } -} - // What the announcer says reaches the controller as it is sent: the two fields, by their names on the // wire, and an older announcement without them reads as not said. func TestTheAnnouncerSaysWhichDirectoriesAreModules(t *testing.T) { diff --git a/cmd/mesh-controller/over_budget.go b/cmd/mesh-controller/over_budget.go deleted file mode 100644 index 03c578ff..00000000 --- a/cmd/mesh-controller/over_budget.go +++ /dev/null @@ -1,125 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads -// mesh-delivery's `times` and raises the warning `delivery..over-budget` when the newest delivery of a -// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR -// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class -// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery. - -// probeBudgetsID is the probe that reads the delivery times. -const probeBudgetsID = "D16" - -// kindOverBudget is what a delivery over its class's budget raises. -const kindOverBudget = "over-budget" - -// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it); -// the probe raises nothing for another class. -var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute} - -// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it. -type timesAnswer struct { - Classes []timesClass `json:"classes"` -} - -// timesClass is one class's line of `times`. -type timesClass struct { - Class string `json:"class"` - Latest *timesLatest `json:"latest,omitempty"` -} - -// timesLatest is the newest delivery of a class whose delivery time is known. -type timesLatest struct { - ID string `json:"id"` - TookMS int64 `json:"took_ms"` - Longest string `json:"longest,omitempty"` - Landed string `json:"landed,omitempty"` -} - -// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or -// none answers (D3 says that one). -func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) { - if !held { - return nil, nil - } - raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{}) - if errors.Is(err, link.ErrNothingServes) { - return nil, nil - } - if err != nil { - return nil, err - } - var a timesAnswer - if err := json.Unmarshal(raw, &a); err != nil { - return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err) - } - return &a, nil -} - -// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget: -// strictly longer, so a delivery of exactly its budget is within it. -func overBudgetObservations(a *timesAnswer) []conditions.Observation { - if a == nil { - return nil - } - var out []conditions.Observation - for _, c := range a.Classes { - budget, ok := deliveryBudgets[c.Class] - if !ok || c.Latest == nil { - continue - } - took := time.Duration(c.Latest.TookMS) * time.Millisecond - if took <= budget { - continue - } - longest := c.Latest.Longest - if longest == "" { - longest = "not known" - } - out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget, - Severity: conditions.Warning, - Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+ - "its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took), - humanDuration(budget), longest), - Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest), - Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took), - humanDuration(budget)), - Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+ - "held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+ - "a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest), - Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)), - }) - } - return out -} - -// probeBudgets is D16: the newest delivery of each class lands within its class's budget. -func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - entries, err := d.open.inventory.Catalogued(ctx) - if err != nil { - return nil, err - } - var conn *nats.Conn - if d.js != nil { - conn = d.js.Conn() - } - a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries)) - if err != nil { - return nil, err - } - return overBudgetObservations(a), nil -} diff --git a/cmd/mesh-controller/over_budget_integration_test.go b/cmd/mesh-controller/over_budget_integration_test.go deleted file mode 100644 index c16b24ad..00000000 --- a/cmd/mesh-controller/over_budget_integration_test.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) { - entries := []inventory.Entry{ - {Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}}, - {Manifest: catalogue.Manifest{Module: "gitea"}}, - } - walk := func(modules ...string) inventory.Plan { - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}} - for _, m := range modules { - p.Modules[m] = &inventory.PlanModule{} - } - return p - } - for want, w := range map[string]inventory.Plan{ - inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"), - } { - if got := classOf(w, entries); got != want { - t.Errorf("%v: %s, want %s", w.Modules, got, want) - } - } - if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore { - t.Errorf("the resolver's holder is core: %s", got) - } - now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}} - times := walkTimesAtCut(batch, walk("gitea"), entries, now) - if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) || - times.Class != inventory.ClassLeaf { - t.Fatalf("times at the cut: %+v", times) - } - batch.Delivery.Batch.AtMost = now.Add(-time.Minute) - if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) { - t.Fatalf("a window closed at its maximum: %v", times.WindowClosed) - } - if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil { - t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed) - } -} diff --git a/cmd/mesh-controller/over_budget_test.go b/cmd/mesh-controller/over_budget_test.go deleted file mode 100644 index 9ba16ee4..00000000 --- a/cmd/mesh-controller/over_budget_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of -// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7, -// the leaf's budget as ADR 0304 decision 4 changed it). -func TestADeliveryOverItsBudgetIsLoud(t *testing.T) { - a := ×Answer{Classes: []timesClass{ - {Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(), - Longest: "judgement 2m40s"}}, - {Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(), - Longest: "build 1m"}}, - {Class: "unclassed", Latest: ×Latest{ID: "x@y", TookMS: time.Hour.Milliseconds()}}, - }} - obs := overBudgetObservations(a) - if len(obs) != 1 { - t.Fatalf("one class over its budget, got %d: %+v", len(obs), obs) - } - o := obs[0] - if o.Key() != "delivery.leaf.over-budget" || !strings.Contains(o.Summary, "judgement 2m40s") || - !strings.Contains(o.Summary, "novox/mesh-catalog@abc") { - t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary) - } - // The next leaf delivery within its budget clears it: the probe raises nothing for the class. - a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()} - if obs := overBudgetObservations(a); len(obs) != 0 { - t.Fatalf("a delivery of exactly its budget is within it: %+v", obs) - } - a.Classes[1].Latest.TookMS = (10*time.Minute + time.Second).Milliseconds() - if obs := overBudgetObservations(a); len(obs) != 1 || obs[0].Key() != "delivery.core.over-budget" { - t.Fatalf("a core delivery over ten minutes: %+v", obs) - } - if obs := overBudgetObservations(nil); obs != nil { - t.Fatal("no answer raises nothing") - } - // No delivery of a class with a known time yet: nothing said of it. - if obs := overBudgetObservations(×Answer{Classes: []timesClass{{Class: inventory.ClassLeaf}}}); len(obs) != 0 { - t.Fatalf("a class with no delivery: %+v", obs) - } -} - -// The probe's question is one the controller's grant names: a question the bus refuses checks nothing. -func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) { - found := false - for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner { - found = found || (v.Seat == catalogue.DeliverySeat && v.Verb == "times") - } - if !found { - t.Fatal("the grant does not name mesh-delivery.times") - } -} - -// A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver, -// leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut. - -// What each machine of the rest was sent is kept only for the machines the send reached. -func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) { - got := restOf([]string{"ace", "g14"}, []string{"ace", "shanks"}, map[string]inventory.SentDeclaration{"ace": {Digest: "d1"}}) - if len(got) != 1 || got["ace"].Digest != "d1" { - t.Fatalf("rest: %+v", got) - } - if restOf([]string{"g14"}, []string{"ace"}, nil) != nil { - t.Fatal("no machine reached: nothing kept") - } -} diff --git a/cmd/mesh-controller/own_grants.go b/cmd/mesh-controller/own_grants.go index 8964a078..5588e2f9 100644 --- a/cmd/mesh-controller/own_grants.go +++ b/cmd/mesh-controller/own_grants.go @@ -60,9 +60,11 @@ func startWithOwnGrants(ctx context.Context, c ownGrantsCarrier, then func()) { // ownGrantsOf is the carrier of the serving controller: the grants step's reading and its grants-only send. func ownGrantsOf(open *stores) ownGrantsCarrier { return ownGrantsCarrier{ - behind: func(ctx context.Context) (string, bool, error) { return brokerBehindOf(ctx, open, nil) }, + behind: func(ctx context.Context) (string, bool, error) { return brokerBehind(ctx, open, nil) }, send: func(ctx context.Context, holder string) error { - _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}) + // The user list is a file of the bus module's composition (novox/hq ADR 0539 clause 7): composed, and the + // assigner writes the bus node's want for that revision at once. + _, err := composeModules(ctx, open, []string{holder}) return err }, } diff --git a/cmd/mesh-controller/own_grants_test.go b/cmd/mesh-controller/own_grants_test.go index d7a8ea4f..1caaf51c 100644 --- a/cmd/mesh-controller/own_grants_test.go +++ b/cmd/mesh-controller/own_grants_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "strings" "testing" ) @@ -20,54 +19,3 @@ func TestAStartingControllerSendsItsGrantsBeforeItWrites(t *testing.T) { t.Fatalf("the start did %v", order) } } - -// 1800 (3), (4): no send when the list is current or there is no bus with a list; one that cannot be read or sent is -// said once, and the controller goes on. -func TestAStartingControllerSendsNothingWhenTheListIsCurrentAndGoesOnWhenRefused(t *testing.T) { - sends := 0 - send := func(context.Context, string) error { sends++; return nil } - for _, c := range []ownGrantsCarrier{ - {behind: func(context.Context) (string, bool, error) { return "novox", false, nil }, send: send}, - {behind: func(context.Context) (string, bool, error) { return "", true, nil }, send: send}, - } { - if said := c.carry(t.Context()); said != "" || sends != 0 { - t.Fatalf("a current list sent %d and said %q", sends, said) - } - } - refused := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "novox", true, nil }, - send: func(context.Context, string) error { return errNotGrantsOnly }} - went := false - startWithOwnGrants(t.Context(), refused, func() { went = true }) - if !went { - t.Fatal("a refused send stopped the controller") - } - if said := refused.carry(t.Context()); !strings.Contains(said, "could not be sent to novox") || - !strings.Contains(said, errNotGrantsOnly.Error()) { - t.Fatalf("a refused send is said as %q", said) - } - unread := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "", false, errors.New("no store") }, - send: send} - if said := unread.carry(t.Context()); !strings.Contains(said, "cannot be read") || sends != 0 { - t.Fatalf("an unread list: sent %d, said %q", sends, said) - } -} - -// 1800 (2): the start's send is the grants step's: composed with every build of the bus's machine kept at the one it -// runs, so the declaration differs from its last only by the user list (and is refused unsent otherwise, -// errNotGrantsOnly, release.go), whatever a running plan registered since. -func TestAStartingControllersSendKeepsEveryBuildOfTheBusMachine(t *testing.T) { - was := sendRollout - t.Cleanup(func() { sendRollout = was }) - var kept bool - var named []string - sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) { - kept, named = everyBuildKept(ctx, "novox"), names - return names, nil - } - if err := ownGrantsOf(nil).send(t.Context(), "novox"); err != nil { - t.Fatal(err) - } - if !kept || len(named) != 1 || named[0] != "novox" { - t.Fatalf("the start's send keeps every build: %v, names %v", kept, named) - } -} diff --git a/cmd/mesh-controller/own_path_by_claim_test.go b/cmd/mesh-controller/own_path_by_claim_test.go deleted file mode 100644 index 0b02f7ac..00000000 --- a/cmd/mesh-controller/own_path_by_claim_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The controller finds the delivery's owner by the claim of the seat mesh-delivery, never by the module's name -// (novox/hq ADR 0437 §3, issue 1210). The gate is a module named mesh-delivery-gate; this is only a fixture name. - -func claimingTheSeat(module string) inventory.Entry { - return inventory.Entry{Manifest: catalogue.Manifest{Module: module, - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}} -} - -func namedOnly(module string) inventory.Entry { - return inventory.Entry{Manifest: catalogue.Manifest{Module: module}, On: []string{"anchor"}} -} - -// 1210/AC1: a module claiming the seat is on the controller's own path whatever its name, awaiting the delivery's -// word as today's module does; a module named mesh-delivery that claims no seat is not on it. -func TestTheModuleClaimingTheSeatIsOnTheControllersOwnPath(t *testing.T) { - gate := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")} - own := ownPathOf(gate) - if _, on := own["mesh-delivery-gate"]; !on { - t.Error("the module claiming the seat is not on the controller's own path") - } - if _, on := own["mesh-delivery"]; on { - t.Error("a module named mesh-delivery that claims no seat is on the controller's own path") - } - if d := awaitsFor(gate, []string{"gitea", "plex"}); d == nil || d.Awaits != catalogue.DeliverySeat { - t.Errorf("an ordinary walk awaits %+v with the gate holding the seat", d) - } - if d := awaitsFor(gate, []string{"mesh-delivery-gate", "gitea"}); d != nil { - t.Errorf("a walk moving the gate awaits %+v; the gate cannot gate itself", d) - } - if d := awaitsFor(gate, []string{"mesh-delivery", "gitea"}); d == nil { - t.Error("a walk moving a module named mesh-delivery that claims no seat starts at once") - } - if !ownWalk(walkOf("novox/one", "notes", "mesh-delivery-gate"), own) || ownWalk(walkOf("novox/one", "notes", "mesh-delivery"), own) { - t.Error("a walk is on the own path by the claim of the module it plans") - } - if ownPath([]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}) != true || ownPath([]inventory.Entry{namedOnly("mesh-delivery")}) { - t.Error("a merge touches the own path by the claim of the module it touches") - } - if classOfModules([]string{"mesh-delivery-gate"}, gate) != inventory.ClassCore { - t.Error("the module claiming the seat is not core") - } -} - -// 1210/AC2: the merge gate's rule of the module claiming the seat is one reading; a module claiming no delivery -// seat gets its ordinary rule. -func TestTheModuleClaimingTheSeatGetsTheDeliverysGateRule(t *testing.T) { - withTheGatesTimes(t) - claimant := claimingTheSeat("mesh-delivery-gate").Manifest - r := gateRuleOf("mesh-delivery-gate", claimant) - if r.passes != 1 || r.within != 0 || !strings.HasPrefix(r.said, "mesh-delivery's rule: one reading (novox/hq ADR 0351)") { - t.Errorf("the gate's rule is %+v", r) - } - r = gateRuleOf("mesh-delivery", catalogue.Manifest{Module: "mesh-delivery"}) - if !strings.HasPrefix(r.said, "a leaf module's rule") { - t.Errorf("a module named mesh-delivery claiming no seat gets %q", r.said) - } -} - -// 1210/AC3: with both today's module and the gate claiming the seat (R3d's group), both are on the own path and -// both get the delivery's rule. -func TestBothClaimantsAreOnTheOwnPathAndGetTheRule(t *testing.T) { - withTheGatesTimes(t) - both := []inventory.Entry{claimingTheSeat("mesh-delivery"), claimingTheSeat("mesh-delivery-gate")} - own := ownPathOf(both) - for _, m := range []string{"mesh-delivery", "mesh-delivery-gate"} { - if _, on := own[m]; !on { - t.Errorf("%s is not on the own path", m) - } - if r := gateRuleOf(m, claimingTheSeat(m).Manifest); r.passes != 1 || !strings.HasPrefix(r.said, "mesh-delivery's rule") { - t.Errorf("%s gets %+v", m, r) - } - if d := awaitsFor(both, []string{m, "gitea"}); d != nil { - t.Errorf("a walk moving %s awaits %+v", m, d) - } - } -} - -// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that -// announce a merge from it. -func theDeliveryClaims(context.Context) ([]inventory.Entry, error) { - return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil -} diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index b2c63231..1bc0b7ac 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -411,13 +411,6 @@ func editDistance(a, b string) int { // refusal. Returns what it said, or nothing when the row no longer waited. func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string { inv := open.inventory - ctx, release, err := holdNodes(ctx, open, []string{p.Node}) - if err != nil { - // Left waiting: the next tick tries again. - return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node, - p.Node, err) - } - defer release() claimed, err := inv.ClaimPending(ctx, p.ID) if err != nil { return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err) @@ -435,9 +428,9 @@ func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignme case strings.Contains(answer, "already runs"): note = fmt.Sprintf("%s already runs %s: the pending assignment is met", p.Node, p.Module) default: - // The same as a person's assign: nothing is sent by this act. - note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); `push %s` sends it", - p.Node, p.Module, by, clock(p.Since), p.Node) + // The same as a person's assign: composed, and its want follows by the assigner's rules. + note = fmt.Sprintf("%s is assigned %s: build %s registered it (pending since %s); composed, its want follows", + p.Node, p.Module, by, clock(p.Since)) } settled, serr := inv.SettlePending(ctx, p.ID, inventory.PendingApplying, state, note) if serr != nil { diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go deleted file mode 100644 index 306ae903..00000000 --- a/cmd/mesh-controller/pending_integration_test.go +++ /dev/null @@ -1,606 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "errors" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func failedBuild(t *testing.T, open *stores, id string) { - t.Helper() - if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa", - "", "boom")); err != nil { - t.Fatal(err) - } -} - -func aCatalogueMesh(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - return open -} - -// mergeAdding keeps the build request a merge adding a module asked before R4e (issue 300): nothing asks one since (ADR -// 0501 §6), and the pending assignments still read the ones kept. These tests are of what an assignment does with one. -func mergeAdding(t *testing.T, open *stores, dir, commit string) { - t.Helper() - if err := open.inventory.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: "b-" + dir, - Repository: "novox/mesh-catalog", Seat: "git", Path: dir, Ref: "main", Commit: commit, For: "merge", - At: time.Now().UTC()}); err != nil { - t.Fatal(err) - } -} - -func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - - requests, err := open.inventory.RequestsNamed(ctx, "sensors") - if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { - t.Fatalf("the merge's build request: %+v %v", requests, err) - } - - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil { - t.Fatalf("an assignment while its build runs was refused: %v", err) - } - t.Logf("assign laptop sensors, while its build runs:\n%s", said) - for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", - "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a module not registered was assigned") - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { - t.Fatalf("pending: %+v", pending) - } - if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { - t.Fatalf("a second assignment: %q %v", again, err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if asked.well() { - t.Fatal("status called the mesh well while an assignment waits") - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc meshStatus - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { - t.Fatalf("status says pending %+v", doc.Pending) - } - - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("the build registered sensors and the pending assignment was not made") - } - pending = pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingApplied || - !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { - t.Fatalf("pending after the build: %+v", pending) - } -} - -func TestAStatusReadSettlesNothing(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - for range 2 { - if _, err := theThreeQuestions(ctx, open); err != nil { - t.Fatal(err) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { - t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) - } - said := settlePending(ctx, open, time.Now()) - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { - t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) - } -} - -func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { - t.Fatal(err) - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingExpired { - t.Fatalf("pending after a failed build: %+v", pending) - } - // Its words since R4g (whyNotBuilt): the build, when it was heard, and that it failed (issue 1938: kept from an older - // run until the inventory changed beneath it). - for _, want := range []string{"expired", "modules/sensors in novox/mesh-catalog; its last build, b-modules/sensors at", ", failed: go build", - "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { - if !strings.Contains(pending[0].Note, want) { - t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a failed build's module was assigned") - } - - settlePending(ctx, open, time.Now()) - key := pendingObservation(pending[0]).Key() - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found || c.Kind != kindPendingEnded { - t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) - } - if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { - t.Fatalf("status does not hold the open condition: %v", err) - } - - _, err = assign(ctx, open, "laptop", "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("a module whose build failed: %v", err) - } - for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", - "failed: go build: undefined: sensorsRead", "registered from the builder's done key", "novox/mesh-catalog (path modules/sensors)"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "taken back") { - t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the assignment was taken back") - } - // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. - asked, err := theThreeQuestions(ctx, open) - if err != nil || pendingOpen(asked.pending) { - t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) - } - for _, c := range asked.conditions { - if c.Kind == kindPendingEnded { - t.Fatalf("status still holds the condition: %+v", c) - } - } -} - -func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "boom")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - key := pendingObservation(pendingOf(t, open)[0]).Key() - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the module was assigned") - } -} - -func TestAFailedAskIsNotABuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { - t.Fatal(err) - } - if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { - t.Fatal(err) - } - _, err := assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { - t.Fatalf("a build not handed over: %v", err) - } - // Heard first, the outcome stands: marking it afterwards changes nothing. - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { - t.Fatal(err) - } - _, err = assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { - t.Fatalf("an outcome heard after a failed wait: %v", err) - } -} - -func TestAPlansBuildRequestIsThePlans(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - // A walk asks no build, so it leaves no build request (askModule takes the module from its key): the plan's app is - // taken from the key at the commit the plan carries. - requests, err := open.inventory.RequestsNamed(ctx, "app") - if err != nil || len(requests) != 0 { - t.Fatalf("a request was recorded for the plan: %+v %v", requests, err) - } - recent, err := open.inventory.RecentPlans(ctx, 5) - if err != nil || len(recent) != 1 { - t.Fatalf("the plans: %+v %v", recent, err) - } - if app := recent[0].Modules["app"]; app == nil || !app.Key || app.KeyAt != "c1aaaaaaaa" { - t.Fatalf("the plan's app is not taken from its key at the commit it carries: %+v", app) - } -} - -func TestABuildBeingRegisteredIsInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { - t.Fatal(err) - } - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { - t.Fatalf("a build being registered: %q %v", said, err) - } - settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) - p := pendingOf(t, open) - if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { - t.Fatalf("past the grace: %+v", p) - } -} - -func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) - asked := asksWithPaths(t) - _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("an unknown module: %v", err) - } - for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", - "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", - "asks for nothing: the controller asks no build"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - if strings.Contains(err.Error(), "merge") { - t.Fatalf("the refusal claims a merge it never looked at: %v", err) - } - if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { - t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) - } -} - -func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - _, err := assign(ctx, open, "laptop", "known", "sensors") - if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { - t.Fatalf("an act naming a module in flight: %v", err) - } - if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { - t.Fatal("an act refused whole was made in part") - } -} - -func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "withdrawn") { - t.Fatalf("unassign of a pending assignment: %q %v", said, err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn assignment was made") - } -} - -func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - p := pendingOf(t, open)[0] - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - _, err := unassign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { - t.Fatalf("unassign of a claimed pending assignment: %v", err) - } - if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { - t.Fatalf("the claim was overridden: %+v", got) - } - // Pending(zero) is the waiting ones alone (review point 7). - if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { - t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) - } - - // Left by a controller that stopped: back to waiting, since sensors is not assigned. - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { - t.Fatalf("a stale claim was not released: %+v", got) - } - - // Withdrawn, then the build registers it: applyPending finds nothing to claim. - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { - t.Fatalf("a withdrawn pending assignment was made: %s", line) - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn pending assignment was assigned") - } - - // A stale claim whose module was assigned is applied. - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { - t.Fatal("claim") - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - rows, _ := inv.PendingFor(ctx, "anchor", "sensors") - if len(rows) != 1 || rows[0].State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", rows) - } -} - -func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - for _, r := range []inventory.BuildRequest{ - {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", - For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, - {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", - For: "merge", At: time.Now()}, - } { - if err := inv.RecordBuildRequest(ctx, r); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"lost", "heard"} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) - - settlePending(ctx, open, time.Now().Add(buildRequestBound)) - if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { - t.Fatal("a module registered meanwhile was not assigned at the next tick") - } - byModule := map[string]inventory.PendingAssignment{} - for _, p := range pendingOf(t, open) { - byModule[p.Module] = p - } - if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { - t.Fatalf("a build that said nothing: %+v", p) - } - if p := byModule["heard"]; p.State != inventory.PendingApplied { - t.Fatalf("a module registered meanwhile: %+v", p) - } - - // Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open - // ones never are left out, nor one whose condition is still open (second review, bug B). - if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { - t.Fatal(err) - } - later := time.Now().Add(inventory.KeptFor + time.Hour) - settlePending(ctx, open, later) - rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor)) - if err != nil { - t.Fatal(err) - } - for _, r := range rows { - if !r.Open() && (r.Raised == nil || r.Cleared != nil) { - t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r) - } - } - for _, r := range rows { - if r.Module == "heard" { - t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r) - } - } -} - -func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-modules/sensors") - settlePending(ctx, open, time.Now()) - row := pendingOf(t, open)[0] - - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) { - t.Fatal("a raised row was pruned, or its condition closed, while the condition was open") - } - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - // No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared. - if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil { - t.Fatalf("a cleared row is not kept, saying so: %v", err) - } - - // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. - if _, err := assign(ctx, open, "anchor", "sensors"); err == nil { - t.Fatal("known and not built was not refused") - } - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - q.Node = "anchor" - if !conditionOpen(t, q) { - t.Fatal("not raised") - } - if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if conditionOpen(t, q) { - t.Fatal("a removed machine's condition stayed open") - } -} - -func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - claimed := func(node string) inventory.PendingAssignment { - p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - return p - } - notAssigned, assigned := claimed("laptop"), claimed("anchor") - register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"}) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - - settlePending(ctx, open, time.Now()) - if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying { - t.Fatal("a fresh claim was settled") - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", got) - } - // Released to waiting, and in the same tick made, since sensors is registered now. - if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied || - !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatalf("a stale claim of a module not assigned: %+v", got) - } -} - -// A10 (ADR 0501 §6; issue 1851): an assignment of a module the indexer holds and the catalogue does not yet waits for -// the builder's done key of it, asking nothing, and is made when that key registers the module. -func TestAnAssignmentWaitsForTheKeyAndAsksNothing(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - noBuildAsked(t) - b := aBuilderOnTheBus(t) - fp := fpOf('a') - manifest := b.indexed(t, "sensors", "1", "novox/mesh-catalog", "modules/sensors", "3da80a4b00aa0000", fp) - b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildBuilding) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil || !strings.Contains(said, "registered from the builder's done key") || !strings.Contains(said, "kept as pending") { - t.Fatalf("A10: the assignment of a module whose key is coming: %q %v", said, err) - } - if p := pendingOf(t, open); len(p) != 1 || p[0].Build != keyPendingPrefix+"sensors" { - t.Fatalf("A10: pending %+v", p) - } - settlePending(ctx, open, time.Now().Add(time.Hour)) - if p := pendingOf(t, open); p[0].State != inventory.PendingWaiting { - t.Fatalf("A10: a pending assignment waiting for a building key expired: %+v", p) - } - b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildDone) - takeNewModules(ctx, open, "sensors") - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatalf("A10: the key registered sensors and the pending assignment was not made: %+v", pendingOf(t, open)) - } -} diff --git a/cmd/mesh-controller/person_wait_integration_test.go b/cmd/mesh-controller/person_wait_integration_test.go deleted file mode 100644 index 72bb5dd6..00000000 --- a/cmd/mesh-controller/person_wait_integration_test.go +++ /dev/null @@ -1,165 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - asLink := func(r inventory.ResourceHealth) link.ResourceHealth { - return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, - Reason: r.Reason, Account: r.Account} - } - account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) - openNow := func() []conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - say(account, unit) - say(account, unit) - list := openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { - t.Fatalf("raised %+v; want lights' relogin-needed alone", list) - } - c := list[0] - if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || - !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { - t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) - } - // After the new login the account is healthy, and the unit, still failing, is the module's own fault. - account.State, account.Reason = link.StateHealthy, "" - say(account, unit) - list = openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { - t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) - } - unit.State, unit.Reason = link.StateHealthy, "" - say(account, unit) - if list = openNow(); len(list) != 0 { - t.Fatalf("after the unit runs: %+v; want nothing open", list) - } -} - -func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), - Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, - Reason: "down"}}}} - return f, err - } - gateEvery, gateBound = 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || - !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { - t.Fatal("late's build is not marked failed at its gate") - } - current, _ := inv.CurrentBuilds(ctx) - if current["app"].Commit != "c2" || current["late"].Commit != "c1" { - t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, - current["late"].Commit) - } - if !strings.Contains(p.Note, "passed on their own and kept: app") { - t.Errorf("the walk does not say what kept its pass: %s", p.Note) - } -} - -func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - store := conditions.NewInMemory() - told := &conditions.Told{} - k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) - t.Cleanup(func() { k.Close(context.Background()) }) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", - Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} - account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", - Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} - say(unit) - say(unit) // lights not working - say(account, unit) - say(account, unit) // now it waits for a login - var resolved []string - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { - time.Sleep(20 * time.Millisecond) - for _, e := range told.Said() { - if e.Change == conditions.ChangeCleared { - resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) - // The clearing line is held to the plain rule too (ADR 0253). - w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation, - Resolved: e.Condition.Resolved, Needs: e.Condition.Needs} - if why, ok := conditions.PlainWords(w, "laptop"); !ok { - t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why) - } - } - } - } - if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { - t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) - } -} diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index eeb3a77d..2355abfd 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -3,9 +3,7 @@ package main import ( "strings" "testing" - "time" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -29,7 +27,6 @@ func userUnit(module, account string) inventory.ResourceHealth { // Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as // before, so that a fault is never excused. func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) notInGroup := relogin("lights", "operator") notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why" systemUnit := userUnit("lights", "operator") @@ -71,45 +68,9 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { } } -// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module -// with a container down beside it is not yet healthy, as before. -func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - now := time.Now() - since := now.Add(-time.Minute) - account := relogin("lights", "operator") - f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, - Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}, - groupsAdded: map[string]bool{"lights": true}} - if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") { - t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why) - } - // **Only a move that put the account in a new group is excused** (issue 318 review): a later build that - // adds no group did not bring the wait, and is judged as before. - f.groupsAdded["lights"] = false - if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet { - t.Fatalf("a wait the move did not bring reads %v %q; want not yet", h, why) - } - f.groupsAdded["lights"] = true - h := f.health["laptop"] - h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", - Target: "lights", State: link.StateUnhealthy, Reason: "down"}) - f.health["laptop"] = h - if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet { - t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why) - } -} - -// The module's condition says the wait in one sentence for a person, and clears on the first statement that -// no longer says it — said as the module's own fault when its unit still fails after the new login. - -// A module that is healthy on its own keeps its pass when another module of the same send fails at the -// bound: it is neither put back nor left without a verdict for every other walk to wait on. - // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers // no button, since nothing but their own new login can do it. func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ relogin("openrazer", "operator"), userUnit("openrazer", "operator")}) plainExample(t, o, "openrazer waits for a new login on g14", @@ -137,34 +98,3 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { t.Errorf("the kind's wording: %+v", w) } } - -// When a module not working turns into one waiting for a new login, and back, the clearing line says what it -// became, never that it works again (issue 318 review). - -// Which moves add an account group, read from the builds' manifests (issue 318 review). -func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - user := func(groups ...any) catalogue.Manifest { - return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", - "name": "operator", "groups": groups}}} - } - none := catalogue.Manifest{Module: "lights"} - for _, c := range []struct { - name string - from catalogue.Manifest - had bool - to catalogue.Manifest - addsSome bool - }{ - {"a group added", none, true, user("lights"), true}, - {"the same group kept", user("lights"), true, user("lights"), false}, - {"a second group added", user("lights"), true, user("lights", "video"), true}, - {"a group taken away", user("lights", "video"), true, user("lights"), false}, - {"new to the machine, with a group", none, false, user("lights"), true}, - {"new to the machine, no group", none, false, none, false}, - } { - if got := addsAccountGroups(c.from, c.had, c.to); got != c.addsSome { - t.Errorf("%s: adds %v; want %v", c.name, got, c.addsSome) - } - } -} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 184e5d69..b6b73bdb 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -10,7 +10,6 @@ package main import ( "fmt" - "sort" "strings" "time" @@ -95,13 +94,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ "the mesh until it does. A machine asleep is normal.", m, m), Resolved: "Tools on " + m + " answer again"} }), - kindAwaitingPush: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - return words{Headline: m + " waits for new instructions", - Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.", - conditions.Capital(m)), - Resolved: m + " got its new instructions"} - }), "declaration-refused": worded(func(o conditions.Observation) words { if o.Scope == conditions.ScopeMesh { return words{Headline: "No machine can get new instructions", @@ -203,33 +195,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ "send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m), Resolved: "Resolved: the big files for " + m + " reach the bus again"} }), - kindDeclarationNearLimit: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - if o.Severity == conditions.Urgent { - return words{Headline: m + " cannot be sent its instructions", - Needs: "make the instructions for " + m + " smaller, for example by taking a module off it.", - Explanation: fmt.Sprintf("The instructions for %s have grown past the largest message the bus takes, so "+ - "nothing new reaches %s until they are smaller.", m, m), - Resolved: "Resolved: the instructions for " + m + " fit the bus again"} - } - return words{Headline: "The instructions for " + m + " are nearly too big", - Needs: "do not add more to " + m + " until this is dealt with; the details give the size.", - Explanation: fmt.Sprintf("The instructions for %s are close to the largest message the bus takes. "+ - "One more module or secret may stop anything new from reaching %s.", m, m), - Resolved: "Resolved: the instructions for " + m + " have room again"} - }), - kindCoreDowngrade: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - component := idPart(o, 0) - if o.Scope != conditions.ScopeCore || !guarded(component) { - component = "core component" - } - return words{Headline: m + " keeps a newer " + component + " than the mesh holds", - Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m), - Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+ - "the mesh's sends keep the newer one there rather than put it back.", component, m), - Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"} - }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, @@ -435,33 +400,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.", Resolved: "Resolved: the delivery moves again"} }), - kindBatchNotCut: worded(func(o conditions.Observation) words { - return words{Headline: "Merged changes are not being delivered", - Explanation: "Merges collected for one delivery should have been planned and were not. Nothing is lost.", - Resolved: "Resolved: the merged changes are being delivered"} - }), - kindBatchBehindWalk: worded(func(o conditions.Observation) words { - return words{Headline: "Merged changes wait behind a slow delivery", - Explanation: "Merges collected for the next delivery wait for the delivery before them, which is taking " + - "longer than it should. Nothing is lost.", - Resolved: "Resolved: the merged changes no longer wait"} - }), - kindWalkWaiting: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery is waiting to start", - Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " + - "out) has not let it start yet.", - Resolved: "Resolved: the delivery is no longer waiting"} - }), - kindPlanFailed: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery failed", - Explanation: "A delivery did not finish. Nothing is tried again until you say so.", - Resolved: "Resolved: the delivery is no longer failed"} - }), - kindMovesHeld: worded(func(o conditions.Observation) words { - return words{Headline: "An update waits for you", - Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.", - Resolved: "Resolved: the update is no longer waiting"} - }), kindImageMissing: worded(func(o conditions.Observation) words { return words{Headline: "A module is not updated", Explanation: "The new version of a module names a container image that the mesh's store does not hold, so it " + @@ -474,30 +412,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", Resolved: "Resolved: the build has its result"} }), - kindRolledBack: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - module := idPart(o, 0) - if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore { - module = "a module" - } - if o.Scope == conditions.ScopeCore { - return words{Headline: "A core update was put back on " + m, - Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+ - "before was put back and runs.", module, m), - Resolved: "Resolved: the core on " + m + " is healthy"} - } - return words{Headline: conditions.Capital(module) + " update put back on " + m, - Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+ - "before, which runs. The change is not delivered until it is fixed.", module, m), - Resolved: "Resolved: " + module + " on " + m + " is settled"} - }), - kindRollbackFailed: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - return words{Headline: "Putting back an update failed on " + m, - Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m), - Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m), - Resolved: "Resolved: " + m + " runs a good build again"} - }), "release-held": worded(func(o conditions.Observation) words { return releaseHeldWords(nil, o.Also) }), @@ -630,26 +544,12 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "The mesh's message system is being upgraded; some things pause until it is done.", Resolved: "The bus upgrade is done"} }), - kindBusStepWaiting: worded(func(o conditions.Observation) words { - return words{Headline: "Sends wait for a bus upgrade", - Needs: "start the bus upgrade " + FromMeshMCPServer, - Explanation: "A new version of the mesh's message system is built, and only a person installs it. Until " + - "then nothing else is sent to the machine that runs it.", - Resolved: "Resolved: the bus upgrade started, and sends go on"} - }), kindBusUpgradeFailed: worded(func(o conditions.Observation) words { return words{Headline: "The bus upgrade failed", Needs: "decide whether to put the bus back to the version before; the details say how.", Explanation: "The upgrade of the mesh's message system did not end healthy in time.", Resolved: "Resolved: the bus is healthy after its upgrade"} }), - kindBackupHeld: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery waits for a backup that failed", - Needs: "fix what stopped the backup, then release the delivery.", - Explanation: "Before a delivery moves a module that keeps data, that data is backed up. A backup failed or " + - "did not answer in ten minutes, so nothing of the delivery is built or sent.", - Resolved: "Resolved: the backup is done and the delivery goes on"} - }), kindStuck: worded(func(o conditions.Observation) words { return words{Headline: "Something in a delivery has stopped moving", Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " + @@ -657,12 +557,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ "said again until it moves.", Resolved: "Resolved: it moves again"} }), - kindSendKeyRefused: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery cannot watch a machine for its report", - Explanation: "The bus refused the note that makes the mesh notice a machine that never reports on a send. " + - "The delivery goes on, and the note is written again at its next look.", - Resolved: "Resolved: the delivery watches the machine for its report again"} - }), kindConsumerBehind: worded(func(o conditions.Observation) words { return words{Headline: "Messages pile up for a listener", Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " + @@ -759,14 +653,6 @@ func humanDuration(d time.Duration) string { return fmt.Sprintf("%d days", int(d.Hours()/24)) } -// repoName is a repository as a person says it: its name without its owner. -func repoName(repository string) string { - if i := strings.LastIndex(repository, "/"); i >= 0 { - return repository[i+1:] - } - return repository -} - // namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more". func namesWords(names []string, most int) string { switch { @@ -780,72 +666,6 @@ func namesWords(names []string, most int) string { return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most) } -// planModules are the modules a walk moves, sorted. -func planModules(p inventory.Plan) []string { - out := make([]string, 0, len(p.Modules)) - for m := range p.Modules { - out = append(out, m) - } - sort.Strings(out) - return out -} - -// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else -// their count and the repository. -func deliveryWhat(modules []string, repository string) string { - if len(modules) > 0 && len(modules) <= 2 { - return namesWords(modules, 2) - } - if len(modules) > 2 { - return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository)) - } - return repoName(repository) -} - -// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery". -func deliveryName(modules []string, repository string) string { - if len(modules) > 0 && len(modules) <= 2 { - return namesWords(modules, 2) + " delivery" - } - return repoName(repository) + " delivery" -} - -// walkWaitingWords explains a walk waiting for its delivery's word. -func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string { - what := deliveryWhat(w.modules, w.repository) - if severity == conditions.Urgent { - return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+ - "delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in)) - } - return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+ - "decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.", - what, humanDuration(in), humanDuration(waitUrgentAfter)) -} - -// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it. -// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying -// where they are given without a channel, and the ask's text drops that (askText). -func waitingNeeds(severity conditions.Severity) string { - if severity == conditions.Urgent { - return "start it, or stop it, " + FromMeshMCPServer - } - return "" -} - -// waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's -// own verbs, approved by the operator (novox/hq ADR 0259). None before the bound. -func waitingActions(plan string, severity conditions.Severity) []conditions.Action { - if severity != conditions.Urgent || plan == "" { - return nil - } - return []conditions.Action{ - {Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, - Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, - {Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, - Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, - } -} - // moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its // account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it. // No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258). @@ -939,186 +759,6 @@ func reloginNeeds(node string) string { return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node) } -// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it. -func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string, - actions []conditions.Action) { - repository, _, _ := strings.Cut(l.ID, "@") - name := repoName(repository) - if l.State == "unannounced" { - // A pull request the forge never announced (novox/hq issue 347): no delivery exists, so there is nothing to - // stop, release or close; a new commit on its branch is announced and checked. - long := "for too long" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - if o.Resolver == conditions.ResolverOperator { - needs = "push a new commit to its branch; the forge announces it and the mesh checks it." - } - pull := "A pull request of " + name - if l.Number > 0 { - pull = fmt.Sprintf("Pull request %s #%d", name, l.Number) - } - return fmt.Sprintf("%s has had no merge check %s", pull, long), - fmt.Sprintf("%s has been open %s on a branch that requires the merge check, and the mesh was never asked "+ - "to check it: the forge never announced it. It cannot merge until it is checked.", pull, long), - fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil - } - if l.State == "unanswered" { - return unansweredWords(l, o) - } - if l.Needs == needsNewHead { - return newHeadWords(l, name) - } - held := l.State - if held == "" { - held = "held" - } - long := "too long" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - if o.Resolver == conditions.ResolverOperator { - // Asked of the operator, approved on a channel that proves who answered (novox/hq ADR 0259); the - // router says where each can be answered, so the words do not. - release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove, - Arguments: map[string]string{"id": l.ID, "why": ""}} - stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove, - Arguments: map[string]string{"id": l.ID, "why": ""}} - switch held { - case "held": - needs, actions = "release it, or stop it, "+FromMeshMCPServer, []conditions.Action{release, stop} - case "ready", "checked": - needs = "merge its pull request, or close it." - case "stopped": - // Already stopped: mesh-delivery.stop refuses it, so neither offer nor ask it (novox/hq issue 1581). - // What ends it is the stalled line's own word, such as a later merge that delivers its modules. - needs = stoppedNeeds(l) - default: - needs, actions = "stop it "+FromMeshMCPServer, []conditions.Action{stop} - } - } - return fmt.Sprintf("Delivery of %s %s %s", name, held, long), - fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long), - fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions -} - -// newHeadWords are the words of a delivery whose cure is its author's new head (novox/hq ADR 0505 §5): it names the -// author and says a new head is needed; it offers no Stop and never calls it the operator's, because the operator -// hears of it only once the author's own watch has failed, after two hours with no new head. -func newHeadWords(l stalledLine, name string) (headline, explanation, resolved, needs string, actions []conditions.Action) { - pull := "A pull request of " + name - if l.Number > 0 { - pull = fmt.Sprintf("Pull request %d of %s", l.Number, name) - } - author := l.Author - if _, plain := conditions.Plain(author); !plain { - author = "its author" // a branch named in code (a dotted name, a number) is not said - } - long := "for a long time" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - return pull + " needs a new head from its author", - fmt.Sprintf("%s cannot merge until %s pushes a new head: its check failed after its Approve, and no new head "+ - "came %s.", pull, author, long), - pull + " has a new head, or is closed", - "ask " + author + " for a new head.", nil -} - -// stoppedNeeds is what ends a delivery that is already stopped, from the stalled line's healer words ("none: a -// merge that delivers its modules ends it — the operator's"): the part that says what ends it. -func stoppedNeeds(l stalledLine) string { - ends := strings.TrimSpace(strings.TrimPrefix(l.H2, "none:")) - ends, _, _ = strings.Cut(ends, " — ") - if ends == "" { - ends = "a later merge that delivers its modules ends it" - } - return "nothing to stop, it is stopped already: " + ends + "." -} - -// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered -// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release -// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look -// at the build queue, where a check that never ran may still wait. -// -// A head whose delivery is held in ready or rejected is said the same way (novox/hq issue 441), and mesh-delivery -// names that state as data (Held). Its check answered once, so the build queue is the wrong place to look: the step -// is a recheck of the delivery, named by its id, which asks the check again (novox/hq issue 451). It is advised, not -// offered as an action, since the controller is not granted the recheck. -func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string, - actions []conditions.Action) { - repository, _, _ := strings.Cut(l.ID, "@") - pull := "A pull request of " + repoName(repository) - if l.Number > 0 { - pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number) - } - long, limit := "for too long", "" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - if d, err := time.ParseDuration(l.Bound); err == nil { - limit = ", past its limit of " + humanDuration(d) - } - check := "merge check" - if said := uncheckedWaitWords(l); len(said) > 0 { - check = "merge check (" + strings.Join(said, ", ") + ")" - } - which := "Its " + check - switch l.Held { - case "ready", "rejected": - which = "Its delivery is " + l.Held + ", but on the forge its " + check - if o.Resolver == conditions.ResolverOperator { - needs = "recheck its delivery, which asks its check again — mesh-delivery.recheck with id " + l.ID + - " and why — or push a new commit to its branch, " + FromMeshMCPServer - } - default: - if o.Resolver == conditions.ResolverOperator { - needs = "push a new commit to its branch, which asks the check again, or see whether its check still " + - "waits in the build queue: mesh-controller.queue." - } - } - return fmt.Sprintf("%s's merge check has not answered %s", pull, long), - fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+ - "It cannot merge until its check answers.", pull, which, long, limit), - fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil -} - -// uncheckedWaitWords are the merge checks an unanswered line waits on, as the operator reads them: from the checks -// given as data, each time in the controller's local zone, as every other time in a message — "mesh/check -// pending since 02:11" — never the UTC time inside mesh-delivery's finished words, which cannot be said again in -// another zone (novox/hq issue 443). A line from a mesh-delivery that gives no such data is said by its words. -func uncheckedWaitWords(l stalledLine) []string { - if len(l.Checks) == 0 { - return l.Waiting - } - out := make([]string, 0, len(l.Checks)) - for _, c := range l.Checks { - switch c.State { - case "never-set": - out = append(out, c.Context+" never set") - case "pending": - out = append(out, c.Context+" pending"+sinceWords(c.Since)) - default: - out = append(out, c.Context+" not answered") - } - } - return out -} - -// sinceWords is " since 02:11" in the controller's local zone, with its day when that is not today ("since 23:05 on -// 9 Oct"), so the time stays absolute; a time not given, or not readable, is said so and never made up. -func sinceWords(since string) string { - at, err := time.Parse(time.RFC3339, since) - if err != nil { - return ", since a time the forge did not say" - } - local, today := at.In(wordsZone()), wordsNow().In(wordsZone()) - if local.YearDay() == today.YearDay() && local.Year() == today.Year() { - return " since " + local.Format("15:04") - } - return " since " + local.Format("15:04 on 2 Jan") -} - // causeWords is a hand-act's cause as a person says it. func causeWords(cause string) string { return strings.NewReplacer(".", " ", "_", " ").Replace(cause) diff --git a/cmd/mesh-controller/plain_words_test.go b/cmd/mesh-controller/plain_words_test.go index 653eaf1c..a4092a29 100644 --- a/cmd/mesh-controller/plain_words_test.go +++ b/cmd/mesh-controller/plain_words_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "regexp" "strings" "testing" @@ -45,51 +44,6 @@ func plainExample(t *testing.T, o conditions.Observation, headline, explanation said, labels, o.Resolved) } -// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for -// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why; -// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no -// popup; past them, the operator starts or stops it from the notification. -func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) { - now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC) - f := calm(now) - f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee", - awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}} - got := watchWaits(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - plainExample(t, got[0], "openrazer delivery waiting to start", - "Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+ - "mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+ - "for you if it still waits after 4 hours.") - if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") { - t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary) - } - - // Past four hours it is urgent, and asks the operator to start or stop it (novox/hq ADR 0259): the plan's - // own verbs, approved, which the controller performs on the warrant. The router says where to answer. - f.waits[0].since = now.Add(-5 * time.Hour) - got = watchWaits(f) - plainExample(t, got[0], "openrazer delivery waiting to start", - "Needs you: start it, or stop it, from the mesh MCP server; this notification cannot do it. The change to openrazer is merged and built, and mesh-delivery (the "+ - "module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+ - "be stuck.", "Start", "Stop") - for i, want := range []string{"go", "stop"} { - a := got[0].Actions[i] - if a.Verb != "mesh-controller.plans" || a.Arguments[want] != "plan-1791454185265004861" || - a.Level != conditions.LevelApprove || a.Arguments["cause"] != conditions.CauseOperatorAnswer { - t.Errorf("%s: %+v", a.Label, a) - } - } - - // Many modules are counted, not listed in the headline. - f.waits[0].modules = []string{"a", "b", "c", "d"} - got = watchWaits(f) - if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") { - t.Errorf("%+v", got[0]) - } -} - // **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the // account's own service manager (exit-code)". Restarting is not an acknowledgement: it is asked of the // operator at the approve level (novox/hq ADR 0259), so a desk click never performs it (ADR 0258). @@ -160,21 +114,6 @@ func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) { "wanted for it. Nothing is broken now.") } -// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been -// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …". -func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) { - got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", - Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}}) - plainExample(t, got[0], "Delivery of hq held for 36 hours", - "Needs you: release it, or stop it, from the mesh MCP server; this notification cannot do it. A delivery of hq has been held for 36 hours, past its limit.", - "Release", "Stop") - for i, verb := range []string{"mesh-delivery.release", "mesh-delivery.stop"} { - if a := got[0].Actions[i]; a.Verb != verb || a.Arguments["id"] != "novox/hq@055550802096" || a.Level != conditions.LevelApprove { - t.Errorf("%+v", a) - } - } -} - // **Every kind the controller raises has plain words**, and its words are plain for a subject of every // shape it is raised with. func TestEveryWordingIsPlain(t *testing.T) { @@ -242,27 +181,6 @@ func TestDataLossOffersNoSilence(t *testing.T) { } } -// **Updates held after a failed release** (2026-10-08): the popup read "Needs you: release them, or leave -// them held." — naming neither what waits nor where it is released. It names the modules and machines, and -// the mesh MCP server. -func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) { - saved := backlogNow - t.Cleanup(func() { backlogNow = saved }) - backlogNow.held = "release-1791457717307061152 failed (failed its gate on g14); what waits is released again by a person" - backlogNow.waiting = map[string][]inventory.CarriedMove{ - "shanks": {{Module: "openrazer"}}, - "g14": {{Module: "openrazer"}, {Module: "sensors"}}, - } - got := backlogObservation() - if len(got) != 1 { - t.Fatalf("raised %d", len(got)) - } - plainExample(t, got[0], "Updates of openrazer and sensors wait for your release", - "Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+ - "sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+ - "They are not delivered until then, and stay held if you leave them.", "Release") -} - // **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any // walk failed, not a release, and a "check" is a pull request's status. So the words say the walk failed, // and never that a release failed or a check did — with the modules and machines named or not. @@ -286,236 +204,3 @@ func TestUpdatesHeldSayTheWalkFailed(t *testing.T) { // mcpVerb is a need that opens with a verb only the mesh MCP server performs (ADR 0258 §1): release, stop, // start, restart, and a restore. var mcpVerb = regexp.MustCompile(`^(release|stop|start|restart|restore)\b`) - -// **A need no notification can answer says where it is answered** (ADR 0258 §1): every wording whose need -// opens with a verb the mesh MCP server performs, and offers no action, ends with FromMeshMCPServer — the -// kinds worded here for every subject shape, and those worded where they are raised. A new kind that misses -// it fails here; release-held did (2026-10-08). -func TestANeedNoNotificationAnswersNamesTheMeshMCPServer(t *testing.T) { - check := func(what string, w words) { - t.Helper() - if w.Needs == "" || len(w.Actions) > 0 || !mcpVerb.MatchString(w.Needs) { - return - } - if !strings.HasSuffix(w.Needs, FromMeshMCPServer) { - t.Errorf("%s needs %q without %q", what, w.Needs, FromMeshMCPServer) - } - } - subjects := []conditions.Observation{ - {Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"}, - {Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"}, - {Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}, - {Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"}, - {Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"}, - {Scope: conditions.ScopeMesh, ID: "release", Also: []string{"g14"}}, - } - for kind, fn := range plainWordings { - for _, s := range subjects { - for _, sev := range []conditions.Severity{conditions.Warning, conditions.Urgent} { - s.Kind, s.Severity, s.Resolver = kind, sev, conditions.ResolverOperator - check(kind+" about "+s.ID, fn(s)) - } - } - } - check("a walk waiting", words{Needs: waitingNeeds(conditions.Urgent)}) - check("a module's failed service", words{Needs: moduleNeeds("g14", - []inventory.ResourceHealth{{Kind: link.KindUnit, Target: "openrazer-daemon.service"}})}) - for _, state := range []string{"held", "ready", "failing"} { - _, _, _, needs, actions := stalledWords(stalledLine{ID: "novox/hq@055550802096", State: state, For: "36h"}, - conditions.Observation{Resolver: conditions.ResolverOperator}) - check("a delivery "+state, words{Needs: needs, Actions: actions}) - } - check("updates held", releaseHeldWords([]string{"openrazer"}, []string{"g14"})) -} - -// **A pull request the forge never announced says what to do about it** (novox/hq issue 347): mesh-delivery says one -// as a stalled line in state `unannounced` — no delivery exists, so there is nothing to stop, release or close — -// and the operator's one act is a new commit on its branch, which the forge announces. -func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) { - l := stalledLine{ID: "novox/hq@bfe82315f42c", Number: 243, State: "unannounced", For: "11m0s", Bound: "10m0s", - H2: "none: the forge never announced it, so there is no delivery to close — the operator's", - Says: "novox/hq#243 is open on main, which requires the merge check, and its head has had no merge check"} - obs := stalledObservations([]stalledLine{l}) - if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator { - t.Fatalf("an unannounced pull request is not the operator's: %+v", obs) - } - o := obs[0] - if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") { - t.Fatalf("it says to %q", o.Needs) - } - if !strings.Contains(o.Headline, "no merge check") || !strings.Contains(o.Headline, "#243") || - !strings.Contains(o.Explanation, "never") { - t.Fatalf("it reads %q / %q", o.Headline, o.Explanation) - } -} - -// **A pull request whose merge check never answered says which check, since when, and what to do** (novox/hq issue -// 438): mesh-delivery says one as a stalled line in state `unanswered`, the line below exactly as its test makes it. -// No delivery of the head is held, so there is nothing to stop, release or close: no action is offered, and the -// operator's acts are a new commit, which asks the check again, or a look at the build queue. -func TestAnUnansweredMergeCheckSaysWhichCheckAndWhatToDo(t *testing.T) { - var l stalledLine - if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+ - `"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],"for":"1h1m0s",`+ - `"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+ - `"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+ - `started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+ - `No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+ - `its check"}`), &l); err != nil { - t.Fatal(err) - } - obs := stalledObservations([]stalledLine{l}) - if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator { - t.Fatalf("an unanswered merge check is not the operator's: %+v", obs) - } - o := obs[0] - t.Logf("headline: %s\nexplanation: %s\nneeds: %s\nresolved: %s", o.Headline, o.Explanation, o.Needs, o.Resolved) - if len(o.Actions) != 0 { - t.Fatalf("it offers an action a head with no delivery cannot take: %+v", o.Actions) - } - if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") || - !strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "build seat") { - t.Fatalf("it says to %q", o.Needs) - } - if o.Headline != "Pull request mesh-controller #212's merge check has not answered for 61 minutes" { - t.Fatalf("its headline reads %q", o.Headline) - } - for _, want := range []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set", - "past its limit of 60 minutes"} { - if !strings.Contains(o.Explanation, want) { - t.Fatalf("its explanation does not say %q: %q", want, o.Explanation) - } - } - if strings.Contains(o.Explanation, "never asked") || strings.Contains(o.Explanation, "never announced") { - t.Fatalf("it says the mesh was never asked, of a head whose check started: %q", o.Explanation) - } - - // A line from a mesh-delivery that names no checks still says the check did not answer, and offers nothing. - var bare stalledLine - if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+ - `"for":"1h1m0s","bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's"}`), - &bare); err != nil { - t.Fatal(err) - } - o = stalledObservations([]stalledLine{bare})[0] - if len(o.Actions) != 0 || !strings.Contains(o.Explanation, "has not answered") || strings.Contains(o.Headline, "Delivery of") { - t.Fatalf("a line naming no checks reads %q / %q, actions %+v", o.Headline, o.Explanation, o.Actions) - } -} - -// **An unanswered merge check's time reaches the operator in their own time, never as raw UTC** (novox/hq issue -// 443): mesh-delivery says each check waited on as data — its context, its state and since when, in RFC 3339 — beside -// the finished words it gave before, and the controller words it in its local zone, as every other time in a message. -// The line is the one mesh-delivery says, as in the report of issue 443, with the checks it now carries. -func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) { - // The operator's zone given through the seam, never by writing time.Local: other tests' goroutines read it, and - // the build seat runs the suite under the race detector. - wasZone, wasNow := wordsZone, wordsNow - wordsZone = func() *time.Location { return time.FixedZone("CEST", 2*60*60) } - wordsNow = func() time.Time { return time.Date(2026, 10, 11, 1, 12, 39, 0, time.UTC) } - t.Cleanup(func() { wordsZone, wordsNow = wasZone, wasNow }) - - var l stalledLine - if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+ - `"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],`+ - `"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},`+ - `{"context":"mesh/repo-check","state":"never-set"}],"for":"1h1m0s",`+ - `"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+ - `"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+ - `started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+ - `No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+ - `its check"}`), &l); err != nil { - t.Fatal(err) - } - o := stalledObservations([]stalledLine{l})[0] - t.Logf("explanation: %s", o.Explanation) - for _, raw := range []string{"2026-10-11T00:11:39Z", "00:11", "UTC"} { - if strings.Contains(o.Explanation, raw) || strings.Contains(o.Headline, raw) { - t.Fatalf("the operator reads %q: %q / %q", raw, o.Headline, o.Explanation) - } - } - for _, want := range []string{"mesh/merge-gate pending since 02:11", "mesh/repo-check never set"} { - if !strings.Contains(o.Explanation, want) { - t.Fatalf("its explanation does not say %q: %q", want, o.Explanation) - } - } - - // A check pending since another day than today says which day, so the time stays absolute. - l.Checks[0].Since = "2026-10-09T21:05:00Z" - if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending since 23:05 on 9 Oct") { - t.Fatalf("a check pending since an earlier day reads %q", o.Explanation) - } - - // A pending check whose time the forge did not say is said so, with no time made up. - l.Checks[0].Since = "" - if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending, since a time the forge did not say") { - t.Fatalf("a pending check without its time reads %q", o.Explanation) - } -} - -// **An unanswered merge check on a ready or rejected delivery is told to recheck it** (novox/hq issue 451): -// mesh-delivery says such a line too (issue 441) and names the delivery's state as data, `held`. That check already -// answered once, so the build queue is the wrong place to look; a recheck of the delivery, named by its id, asks the -// check again. A line with no delivery held keeps the advice of issue 438, and so does a line from a mesh-delivery -// that does not say `held` yet: the controller never guesses the state from the words. -func TestAnUnansweredMergeCheckOnAReadyOrRejectedDeliveryOffersARecheck(t *testing.T) { - line := func(held string) stalledLine { - var l stalledLine - raw := `{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",` + - `"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z"],"for":"1h1m0s","bound":"1h0m0s",` + - `"h2":"none: its delivery is ` + held + `, a state the operator moves — the operator's",` + - `"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its delivery is ` + - held + ` here, but on the forge its check has not answered: mesh/merge-gate pending since ` + - `2026-10-11T00:11:39Z. The pull request waits on that status and nothing asks it again. A recheck of the ` + - `delivery asks its check again, as does a new commit on its branch"` - if held != "" { - raw += `,"held":"` + held + `"` - } - if err := json.Unmarshal([]byte(raw+`}`), &l); err != nil { - t.Fatal(err) - } - return l - } - for _, held := range []string{"ready", "rejected"} { - o := stalledObservations([]stalledLine{line(held)})[0] - t.Logf("%s: %s\nneeds: %s", held, o.Explanation, o.Needs) - if !strings.Contains(o.Needs, "mesh-delivery.recheck") || !strings.Contains(o.Needs, "novox/mesh-controller@c11222026a3b") { - t.Fatalf("a %s delivery's unanswered check is not offered its recheck: %q", held, o.Needs) - } - if strings.Contains(o.Needs, "build queue") || strings.Contains(o.Needs, "mesh-controller.queue") { - t.Fatalf("a %s delivery's unanswered check, which answered once, is sent to the build queue: %q", held, o.Needs) - } - if !strings.Contains(o.Explanation, "Its delivery is "+held) { - t.Fatalf("a %s delivery's explanation does not say its state: %q", held, o.Explanation) - } - if len(o.Actions) != 0 { - t.Fatalf("it offers an action the controller is not granted: %+v", o.Actions) - } - } - // No delivery held, or a mesh-delivery that does not say: the advice of issue 438, and no recheck. - o := stalledObservations([]stalledLine{line("")})[0] - if !strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "recheck") { - t.Fatalf("a line with no delivery held says to %q", o.Needs) - } -} - -// **A delivery already stopped is not asked to be stopped** (novox/hq issue 1581): mesh-delivery.stop refuses a -// stopped delivery, so the condition offers no Stop and says what ends it, in the stalled line's own words. Any -// other held state keeps its Stop. -func TestAStoppedDeliveryIsNotAskedToBeStopped(t *testing.T) { - l := stalledLine{ID: "novox/mesh-catalog@2f6086120c46", State: "stopped", For: "12h19m36s", - H2: "none: a merge that delivers its modules ends it — the operator's", - Says: "a merged change never reached the machines"} - _, _, _, needs, actions := stalledWords(l, conditions.Observation{Resolver: conditions.ResolverOperator}) - if len(actions) != 0 { - t.Fatalf("a stopped delivery offers %v, want no action", actions) - } - if strings.Contains(needs, "stop it") || !strings.Contains(needs, "a merge that delivers its modules ends it") { - t.Fatalf("needs = %q, want what ends it and no ask to stop", needs) - } - l.State = "failing" - _, _, _, needs, actions = stalledWords(l, conditions.Observation{Resolver: conditions.ResolverOperator}) - if len(actions) != 1 || actions[0].Label != "Stop" || !strings.HasPrefix(needs, "stop it") { - t.Fatalf("another held state changed: needs %q, actions %v", needs, actions) - } -} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index bd0a70fe..3d2729bb 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -1,11 +1,8 @@ package main import ( - "bytes" "context" - "encoding/json" "errors" - "flag" "fmt" "io" "os" @@ -17,7 +14,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // working out what one machine should be. @@ -126,43 +122,6 @@ func sortedKeysOf(m map[string]string) []string { return out } -// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR -// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out. -func reportLeftOut(node string, declared sendable) { - for _, m := range declared.LeftOut { - fmt.Printf("%s: %s left out — what the machine holds for it is kept and its containers are "+ - "untouched. %s\n", node, m, declared.leftOutWhy[m]) - } - // And whom it serves nothing, because their identity overflows what the provision keeps (ADR - // 0225): the machine is sent everything else, and the consumer is named. - for _, o := range declared.withheld { - fmt.Printf("%s: %s\n", node, o) - } - // And whom it no longer serves because they are bound elsewhere (novox/hq issue 274). - for _, u := range declared.unbound { - fmt.Printf("%s: %s\n", node, u) - } - // And every contribution its holder could not render, which the machine is sent without (novox/hq - // ADR 0255). - for _, u := range declared.unplaced { - fmt.Printf("%s: %s\n", node, u) - } - // And each file only the provisioner's client fills, left out because the module is not handed here. - for _, u := range declared.handedOnly { - fmt.Printf("%s: %s\n", node, u) - } -} - -// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its -// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly. -func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { - stated, had, err := inv.HealthOf(ctx, node) - if err != nil { - return false, err - } - return had && stated.Contract >= link.BusyContract, nil -} - // provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user // the process of the module running it runs as, empty where no such module is on the machine. func provisionerAccount(plan catalogue.Resolution) string { @@ -181,25 +140,6 @@ func provisionerAccount(plan catalogue.Resolution) string { return "" } -// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares -// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it. -func busyNotSent(modules []catalogue.Manifest, reads bool) []string { - if reads { - return nil - } - var out []string - for _, m := range modules { - for _, r := range m.Resources { - if _, has := r[catalogue.BusyField]; has { - out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+ - "statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+ - "(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract)) - } - } - } - return out -} - // certificateFor is what the mesh certifies about one machine's internal name. // // It reaches across two contexts and reads neither one's store from the other: `inventory` knows @@ -262,212 +202,6 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str return issued, authority.Certificate, nil } -// listensLines is what a person is told about what this module would open, and why — the same -// `why` every listens entry already carries for the firewall it also feeds (novox/hq ADR 0007), so -// deciding whether to assign a module can see what it would open before it opens it, not only -// after. A module with nothing to listen on prints nothing extra, same as today. -func listensLines(m catalogue.Manifest) []string { - var out []string - for _, l := range m.Listens { - if l.Why == "" { - out = append(out, fmt.Sprintf(" listens %d/%s from %s", l.Port, l.At(), l.From)) - continue - } - out = append(out, fmt.Sprintf(" listens %d/%s from %s — %s", l.Port, l.At(), l.From, l.Why)) - } - return out -} - -func planCommand(ctx context.Context, args []string) error { - // One reading of the mesh for every machine this looks at. - ctx = withInputCache(ctx) - set := flag.NewFlagSet("plan", flag.ContinueOnError) - // Because "one resource" does not tell you whether the settings landed. Being able to read - // the file before it is sent is the difference between believing a merge worked and knowing. - show := set.Bool("files", false, "print the files this node would be given") - // The declaration exactly as the node would receive it. For handing to something else -- - // checking it against the host's own parser, most usefully, which is the only way to know - // that what the control plane emits is what the host accepts. - asJSON := set.Bool("json", false, "print the declaration this node would be sent") - // What a push would change, against what the machine was last sent (novox/hq ADR 0217): read - // before sending, so a change that removes, moves or recreates something is seen first. - asDiff := set.Bool("diff", false, "what a push would change on this node, against what it was last sent") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) != 1 { - return errors.New("plan [--files] [--json] [--diff]") - } - args = positionals - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - - plan, settings, err := resolveNode(ctx, open, args[0]) - if err != nil { - return err - } - if len(plan.Modules) == 0 { - fmt.Printf("%s is assigned nothing\n", args[0]) - return nil - } - // Which optional requirements are unbound (novox/hq ADR 0330 §7): beside the document on the - // error stream where the answer is the declaration itself, which carries no such field — the - // node-engine refuses an envelope key it does not know. - if *asDiff || *asJSON { - for _, line := range unboundLines(plan) { - fmt.Fprintf(os.Stderr, "%s:%s\n", args[0], line) - } - } - if *asDiff { - declared, err := declarationFor(ctx, open, args[0], plan, settings) - if err != nil { - return err - } - body, err := declared.Body() - if err != nil { - return err - } - // A module left out is not in the body, so the diff alone would say "nothing would change" for - // a module just assigned whose settings cannot compose — success-shaped silence. Said first, with - // why, as push and the plain plan say it (novox/hq ADR 0163, rule 6). - reportLeftOut(args[0], declared) - return writePlanDiff(ctx, open.inventory, args[0], body) - } - if *asJSON { - declared, err := declarationFor(ctx, open, args[0], plan, settings) - if err != nil { - return err - } - // The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an - // envelope other than the one sent. - body, err := declared.Body() - if err != nil { - return err - } - var indented bytes.Buffer - if err := json.Indent(&indented, body, "", " "); err != nil { - return err - } - fmt.Println(indented.String()) - return nil - } - - // What the machine was last told, by whom and from which assignment generation (novox/hq issue 234), - // beside what it would be told now. A record that cannot be read is said, not taken for none. - if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil { - fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err) - } - // Which modules a push would leave out, and why — said before the plan, since the plan is of - // what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan` - // without --json allocates nothing. - if record, err := open.inventory.NodeByName(ctx, args[0]); err == nil { - left := plan.LeftOut(settings, record.Adopted) - reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left}) - } - // And which of its modules no provider will grant, because the identity overflows the bound of - // what it requires (novox/hq ADR 0225) — said on the machine the remedy is for. - for _, o := range plan.Overflowing() { - fmt.Printf("%s: %s\n", args[0], o) - } - // And a setting that reaches nothing — refused where it is stored, and said here for one - // stored before its definition moved from under it. - for _, m := range plan.Modules { - for _, stray := range catalogue.UnusedSettings(m, settings[m.Module]) { - fmt.Printf("%s: a setting reaches nothing — %s\n", args[0], stray) - } - } - fmt.Printf("%s would run:\n", args[0]) - for _, m := range plan.Modules { - fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module]) - for _, line := range listensLines(m) { - fmt.Println(line) - } - } - // What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is - // one module on the wrong machine, the others still run, and the remedy is to move this one. - for _, u := range plan.Unhostable { - for _, c := range u.Missing { - fmt.Printf(" %-20s not applied — %s\n", u.Module, catalogue.WrongMachine(u.Module, c, args[0])) - } - } - for _, c := range plan.Claims { - fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope) - } - // What this machine depends on that is not on it. Worth saying out loud: it is the only part - // of a node's set that stops working when a *different* machine goes away, and nothing else - // in this output would have told anybody that. - for _, n := range plan.Needs { - fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For) - // And a served key the provider left out, because its machine composes no name it asks for - // (novox/hq ADR 0346 §3). - for _, line := range n.LeftOut { - fmt.Printf(" %s\n", line) - } - } - // What the providers on this machine leave out of what they serve, and why: said where the remedy - // is (novox/hq ADR 0346 §3). - for _, line := range plan.ServedLeftOut(settings) { - fmt.Printf("%s: %s\n", args[0], line) - } - for _, line := range unboundLines(plan) { - fmt.Println(line) - } - // And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its - // container is replaced at once, as before. A statement that cannot be read is said as such. - if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil { - fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err) - } else { - for _, line := range busyNotSent(plan.Modules, reads) { - fmt.Printf("%s: %s\n", args[0], line) - } - } - declared, err := declarationFor(ctx, open, args[0], plan, settings) - if err != nil { - return err - } - resources := declared.Resources - for module, layers := range settings { - for _, layer := range layers { - fmt.Printf(" %-20s settings from %s\n", module, layer.From) - } - } - // What each port reached from the local network resolved to on this machine (novox/hq ADR 0340 §8), - // or that it resolved to none. - for _, p := range declared.localNetwork { - fmt.Println(" " + p.Said()) - } - fmt.Printf("\n%d resource(s)\n", len(resources)) - - if *show { - for _, r := range resources { - content, ok := r["content"].(string) - if !ok { - continue - } - fmt.Printf("\n--- %s ---\n%s", shownAs(r), content) - } - } - return nil -} - -// shownAs is the heading `plan --show` puts over a resource's content. -// -// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the -// machine's — the keys of a JSON document (novox/hq ADR 0102), the region of a hosts file (issue -// 128). Shown under a bare path it reads as the whole file, and a person checking what a take -// replaces would see a hosts file of a dozen lines where the machine keeps thirty. -func shownAs(r map[string]any) string { - heading := fmt.Sprintf("%v %v", r["id"], r["path"]) - if into, ok := r["into"].(string); ok && into != "" { - heading += fmt.Sprintf(" (written into, %s)", into) - } - return heading -} - // servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT // node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh // assigned over the manifest's own, settled with the node's settings layers. @@ -637,14 +371,6 @@ func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) { node, len(lines), strings.Join(lines, "\n ")) } -// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over -// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not — -// which includes every one of a module just assigned — and each now met that was not. Nothing about -// any other node, and nothing that was already true before the act. -func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string { - return unheldChangeTo(shelf, node, before, after, unheldOver(shelf, node, after)) -} - // unheldOver is the node's unmet seat dependencies judged over a set of module names alone. func unheldOver(shelf map[string]catalogue.Manifest, node string, names []string) []catalogue.Unheld { var set []catalogue.Manifest diff --git a/cmd/mesh-controller/plan_announce_copy_test.go b/cmd/mesh-controller/plan_announce_copy_test.go deleted file mode 100644 index 66c0a3d0..00000000 --- a/cmd/mesh-controller/plan_announce_copy_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// heldPublish is a bus that holds a plan-moved publish until it is let go, and keeps the body. -type heldPublish struct { - link.Bus - let chan struct{} - body chan []byte -} - -func (h *heldPublish) PublishSeatEvent(_ context.Context, _, _ string, body []byte) error { - <-h.let - h.body <- body - return nil -} - -func aWalkWithSteps() inventory.Plan { - built := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) - return inventory.Plan{ID: "plan-1", Repository: "novox/one", Commit: "abc", State: "walking", Tier: 1, - Created: built, Updated: built, Revision: 4, - Tiers: [][]string{{"alpha"}, {"beta", "gamma"}}, - Modules: map[string]*inventory.PlanModule{ - "alpha": {State: "built", BuiltAt: &built, Commit: "abc", First: []string{"ace"}}, - "beta": {State: "asked", Why: "waiting"}, - "gamma": {}, - }, - Times: &inventory.PlanTimes{Started: &built}, WalkSaid: inventory.WalkStarted} -} - -func TestAnAnnouncementIsThePlanAsItWasWhenSaidWhatTheWalkDoesNext(t *testing.T) { // 1260/AC1 - p := aWalkWithSteps() - bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)} - sayPlanMoved(t.Context(), bus, nil, p) - - // The walk goes on, writing its steps while the announcement is being made. - for i := 0; i < 2000; i++ { - step := p.Modules["beta"] - step.State, step.Why = "built", "changed after the call" - step.First = append(step.First[:0], "novox") - p.Modules["delta"] = &inventory.PlanModule{State: "asked"} - p.Tiers[1][0] = "changed" - p.Times.Started = nil - delete(p.Modules, "delta") - } - close(bus.let) - - var said inventory.Plan - select { - case body := <-bus.body: - if err := json.Unmarshal(body, &said); err != nil { - t.Fatal(err) - } - case <-time.After(10 * time.Second): - t.Fatal("nothing was announced") - } - if said.Modules["beta"].State != "asked" || said.Modules["beta"].Why != "waiting" || said.Tiers[1][0] != "beta" || - len(said.Modules) != 3 || said.Times == nil || said.Times.Started == nil { - t.Errorf("the announcement carries the walk as it went on, not as it was: %+v", said) - } -} - -func TestAnAnnouncementOfAnUnchangedPlanIsTheBodyItWasBeforeTheCopy(t *testing.T) { // 1260/AC2 - p := aWalkWithSteps() - bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)} - close(bus.let) - sayPlanMoved(t.Context(), bus, nil, p) - var got []byte - select { - case got = <-bus.body: - case <-time.After(10 * time.Second): - t.Fatal("nothing was announced") - } - // What it was: the plan whole, its phases worked out, and what the walk says of itself. - q := p - q.Phases = q.WalkPhases(time.Now(), nil) - want, err := json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), nil, q)}) - if err != nil { - t.Fatal(err) - } - var a, b map[string]any - if json.Unmarshal(got, &a) != nil || json.Unmarshal(want, &b) != nil { - t.Fatal("a body is not JSON") - } - delete(a, "phases") // worked out from the time of the call - delete(b, "phases") - if !reflect.DeepEqual(a, b) { - t.Errorf("the body changed:\n got %s\nwant %s", got, want) - } - if a["walk"] != "plan-1" { - t.Errorf("the walk's own words are not in the body: %s", got) - } -} diff --git a/cmd/mesh-controller/plan_backups.go b/cmd/mesh-controller/plan_backups.go deleted file mode 100644 index 024b2fdc..00000000 --- a/cmd/mesh-controller/plan_backups.go +++ /dev/null @@ -1,453 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sort" - "strings" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A backup before the send** (novox/hq ADR 0404 decisions 2 and 3, design 47, issue 1065). Once a delivery plan -// has started and before it asks its first tier, the controller asks the node-backup holder of every machine that -// runs a moved module with an item protected by a backup to back that module up now, and asks nothing of a builder -// and sends nothing until every one of them answered done. The plan keeps the record (inventory.PlanBackups). - -// backupsWithin is how long a backup is given from its ask to say it is done, and backupsPollEvery how soon the plan -// is looked at again while it waits: variables so a test waits for neither. -var ( - backupsWithin = 10 * time.Minute - backupsPollEvery = 10 * time.Second -) - -// night is one module's last night as a holder's `backed-up` says it (lastNight). -type night struct { - OK bool `json:"ok"` - At time.Time `json:"at"` - Error string `json:"error,omitempty"` -} - -// askBackupNow asks a machine's backup holder to back a module up now; the holder answers when it has started. A -// variable so a test needs no bus. An error is the holder's own words when it refused, or why it could not be asked. -var askBackupNow = func(ctx context.Context, machine, module string) error { - return onTheBus(func(conn *nats.Conn) error { - answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", machine, - map[string]any{"module": module}, 30*time.Second) - if err != nil { - return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: %w", machine, err)} - } - if answer.Error != "" { - return fmt.Errorf("%s", answer.Error) - } - return nil - }) -} - -// backupNotAsked is an ask that could not be made, as opposed to one the holder refused: the step is tried again. -type backupNotAsked struct{ error } - -func (e *backupNotAsked) Unwrap() error { return e.error } - -// readLastNights reads a machine's holder's `backed-up` by module: how each module's last backup went. A variable -// so a test needs no bus. -var readLastNights = func(ctx context.Context, machine string) (map[string]*night, error) { - var out map[string]*night - err := onTheBus(func(conn *nats.Conn) error { - answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", machine, map[string]any{}, 10*time.Second) - if err != nil { - return err - } - if answer.Error != "" { - return fmt.Errorf("%s", answer.Error) - } - var read []struct { - Module string `json:"module"` - LastNight *night `json:"lastNight"` - } - if err := json.Unmarshal(answer.Result, &read); err != nil { - return fmt.Errorf("its answer is not readable: %w", err) - } - out = map[string]*night{} - for _, m := range read { - out[m.Module] = m.LastNight - } - return nil - }) - return out, err -} - -// backupFacts are what the step needs of the mesh: the modules with an item protected by a backup, where each -// runs, and which machines hold node-backup. -type backupFacts struct { - backedUp map[string][]string // module → machines running it - holder map[string]bool -} - -// backupFactsFrom reads the facts from the catalogue's entries and the seat holdings. A module is backed up when -// one of its own items is (catalogue.DataItem.BackedUp: never a cache, `backup: none` or redundancy alone). A -// module whose policy records (ADR 0377) is decided per machine after the build, so every machine running it is -// asked: a superset costs time and never leaves a send without its restore point. -func backupFactsFrom(entries []inventory.Entry, held []catalogue.Held) backupFacts { - f := backupFacts{backedUp: map[string][]string{}, holder: map[string]bool{}} - for _, e := range entries { - for _, it := range e.Manifest.DataItems() { - if it.BackedUp() { - f.backedUp[e.Manifest.Module] = slices.Clone(e.On) - break - } - } - } - for _, h := range held { - if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat { - f.holder[h.Node] = true - } - } - return f -} - -// loadBackupFacts reads the facts from the inventory. The holders are read as `seats` reads them, from the world -// every machine's composition makes (theRestOfTheMesh): the seat_holding table keeps one holder of a node seat, and -// node-backup is held on every machine. -func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts, error) { - return loadBackupFactsFrom(inv.Catalogued, func() ([]catalogue.Held, error) { - shelf, err := inv.Catalogue(ctx) - if err != nil { - return nil, err - } - world, err := restOfTheMeshOf(ctx, inv, shelf, "") - if err != nil { - return nil, err - } - return world.Held, nil - }, ctx) -} - -// loadBackupFactsFrom is loadBackupFacts given where the entries and the holdings come from. A source that fails -// fails the step, to be tried again: a failure never reads as a machine with no holder, which would send without a -// backup. -func loadBackupFactsFrom(entries func(context.Context) ([]inventory.Entry, error), - held func() ([]catalogue.Held, error), ctx context.Context) (backupFacts, error) { - es, err := entries(ctx) - if err != nil { - return backupFacts{}, fmt.Errorf("the backups of the plan could not be worked out: %w", err) - } - h, err := held() - if err != nil { - return backupFacts{}, fmt.Errorf("the backups of the plan could not be worked out: who holds %s: %w", - catalogue.BackupSeat, err) - } - return backupFactsFrom(es, h), nil -} - -// stepBackups takes one step of a plan's backups and says whether the plan may go on to ask its first tier. It -// asks what is not asked, reads how the asked ones went, and records it all in p.Backups; it waits (false, nil) -// while any is pending, and stays stopped (false, nil) once one failed, which part B holds and says. A plan whose -// moved modules have no backed-up item keeps no record and goes on at once. A holder that cannot be asked is tried again -// within the bound and then failed; an error is left for what the step itself cannot do. -func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) { - proceed, err := stepBackupsOnce(ctx, p, facts, now) - if err == nil { - noteBackups(p) - } - return proceed, err -} - -func stepBackupsOnce(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) { - if p.Backups == nil { - b := wantedBackups(*p, facts) - if b == nil { - return true, nil - } - p.Backups = b - } - b := p.Backups - switch b.State { - case inventory.BackupDone: - return true, nil - case inventory.BackupFailed: - return false, nil - } - nights := map[string]map[string]*night{} - for i := range b.Asked { - a := &b.Asked[i] - if a.State != inventory.BackupPending { - continue - } - if a.AskedAt == nil { - asked := now - if a.TriedAt == nil { - a.TriedAt = &asked - } - if err := askBackupNow(ctx, a.Machine, a.Module); err != nil { - var unasked *backupNotAsked - if errors.As(err, &unasked) { - // A holder off the bus is tried again within the bound, by the look and never by the - // "tried again" error path; past it the backup is failed and the plan held. - a.Words = "could not be asked: " + err.Error() - if now.Sub(*a.TriedAt) >= backupsWithin { - a.State = inventory.BackupFailed - } - continue - } - a.State, a.Words = inventory.BackupFailed, err.Error() - a.AskedAt, a.TriedAt = &asked, nil - continue - } - a.AskedAt, a.TriedAt, a.Words = &asked, nil, "" - continue - } - if _, read := nights[a.Machine]; !read { - n, err := readLastNights(ctx, a.Machine) - if err != nil { - n = nil // not read now: the bound below still ends the wait - } - nights[a.Machine] = n - } - if n := nights[a.Machine][a.Module]; n != nil && n.At.After(*a.AskedAt) { - if n.OK { - a.State, a.Words = inventory.BackupDone, "" - } else { - a.State, a.Words = inventory.BackupFailed, n.Error - if a.Words == "" { - a.Words = "the backup failed and did not say why" - } - } - continue - } - if now.Sub(*a.AskedAt) >= backupsWithin { - a.State = inventory.BackupFailed - a.Words = "did not answer in " + wordsOfBound(backupsWithin) - } - } - b.State = backupsStateOf(b.Asked) - switch b.State { - case inventory.BackupDone: - return true, nil - case inventory.BackupPending: - lookAgainForBackups(p.ID) - } - return false, nil -} - -// wantedBackups is the record of backups a plan has to take, or nil for none: one entry per machine with a holder -// running a moved module that has a backed-up item, and the machines with none. -func wantedBackups(p inventory.Plan, facts backupFacts) *inventory.PlanBackups { - var asked []inventory.PlanBackup - noHolder := map[string]bool{} - for _, tier := range p.Tiers { - for _, module := range tier { - for _, machine := range facts.backedUp[module] { - if !facts.holder[machine] { - noHolder[machine] = true - continue - } - asked = append(asked, inventory.PlanBackup{Machine: machine, Module: module, State: inventory.BackupPending}) - } - } - } - if len(asked) == 0 && len(noHolder) == 0 { - return nil - } - sort.SliceStable(asked, func(i, j int) bool { - if asked[i].Machine != asked[j].Machine { - return asked[i].Machine < asked[j].Machine - } - return asked[i].Module < asked[j].Module - }) - b := &inventory.PlanBackups{Asked: asked} - for m := range noHolder { - b.NoHolder = append(b.NoHolder, m) - } - sort.Strings(b.NoHolder) - b.State = backupsStateOf(asked) - return b -} - -// backupsStateOf is the whole of the backups' states: failed if any failed, done if all are (or none are asked). -func backupsStateOf(asked []inventory.PlanBackup) string { - state := inventory.BackupDone - for _, a := range asked { - switch a.State { - case inventory.BackupFailed: - return inventory.BackupFailed - case inventory.BackupPending: - state = inventory.BackupPending - } - } - return state -} - -func wordsOfBound(d time.Duration) string { - if d == 10*time.Minute { - return "ten minutes" - } - if d%time.Minute == 0 { - return fmt.Sprintf("%d minutes", int(d/time.Minute)) - } - return d.String() -} - -var ( - backupLookMu sync.Mutex - backupLooks = map[string]bool{} -) - -// lookAgainForBackups has the serving controller look at the plans again soon, once per waiting plan: a backup ending -// is no event the controller hears, so the wait is looked at by a timer, never by blocking the hold on the plans. -func lookAgainForBackups(plan string) { - backupLookMu.Lock() - if backupLooks[plan] { - backupLookMu.Unlock() - return - } - backupLooks[plan] = true - backupLookMu.Unlock() - time.AfterFunc(backupsPollEvery, func() { - backupLookMu.Lock() - delete(backupLooks, plan) - backupLookMu.Unlock() - askToLook() - }) -} - -// backupsFirst is the step advanceOnce takes before it asks a walk's first tier: whether the plan may go on. -func backupsFirst(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) (bool, error) { - facts := backupFacts{} - if p.Backups == nil { - f, err := loadBackupFacts(ctx, inv) - if err != nil { - return false, err - } - facts = f - } - return stepBackups(ctx, p, facts, time.Now().UTC()) -} - -// backupsNote begins the note of a plan that waits on or is held by its backups. -const backupsNote = "backups: " - -// noteBackups says in the plan's note what its backups wait on, or, held, which backup failed and in what words and -// the verb that asks again; a note of its own that the backups ended is cleared. Not the "tried again" note of a -// failed step: a held plan is not retried (novox/hq ADR 0404 decision 4). -func noteBackups(p *inventory.Plan) { - if p.Backups == nil { - return - } - var pending, failed []string - for _, a := range p.Backups.Asked { - switch a.State { - case inventory.BackupPending: - pending = append(pending, a.Module+" on "+a.Machine) - case inventory.BackupFailed: - failed = append(failed, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words)) - } - } - switch p.Backups.State { - case inventory.BackupPending: - p.Note = backupsNote + "waiting for the backup of " + strings.Join(pending, ", ") - case inventory.BackupFailed: - p.Note = backupsNote + "held — the backup of " + strings.Join(failed, "; ") + "; nothing is asked or sent. " + - "`delivery-backups-again` asks it again" - default: - if strings.HasPrefix(p.Note, backupsNote) { - p.Note = "" - } - } -} - -// waitingOnBackups says a plan has not asked its first tier because its backups are pending or failed: no tier -// late, as S3 and the lines of `plans` read it. -func waitingOnBackups(p inventory.Plan) bool { - return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Backups != nil && - p.Backups.State != inventory.BackupDone -} - -// askBackupsAgain clears the failed backups of a held plan, so the next step asks those again and no other. It -// says which. Refused for a plan with no failed backup. -func askBackupsAgain(p *inventory.Plan, by, why string) ([]inventory.PlanBackup, error) { - if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed { - return nil, fmt.Errorf("%s has no failed backup: there is nothing to ask again", p.ID) - } - var again []inventory.PlanBackup - for i := range p.Backups.Asked { - a := &p.Backups.Asked[i] - if a.State != inventory.BackupFailed { - continue - } - again = append(again, *a) - a.State, a.AskedAt, a.TriedAt, a.Words = inventory.BackupPending, nil, nil, "" - } - p.Backups.State = backupsStateOf(p.Backups.Asked) - p.Note = backupsNote + "asked again by " + by + ": " + why - return again, nil -} - -// backupsAgain is askBackupsAgain for the plan with this id, kept: the verb `delivery-backups-again`. -func backupsAgain(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, []inventory.PlanBackup, error) { - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return inventory.Plan{}, nil, err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return inventory.Plan{}, nil, err - } - again, err := askBackupsAgain(&p, by, why) - if err != nil { - return p, nil, err - } - if err := inv.SavePlan(ctx, &p); err != nil { - return p, nil, err - } - return p, again, nil -} - -// kindBackupHeld is a delivery plan held for a backup that failed or stayed silent; backupHeldProbe its row. -const ( - kindBackupHeld = "backup-held" - backupHeldProbe = "DK" -) - -// observeBackupHolds is one condition per plan held for its backups, with the backups' own words as its evidence. -func observeBackupHolds(plans []inventory.Plan) []conditions.Observation { - var out []conditions.Observation - for _, p := range plans { - if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed { - continue - } - var said, machines []string - for _, a := range p.Backups.Asked { - if a.State == inventory.BackupFailed { - said = append(said, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words)) - machines = append(machines, a.Machine) - } - } - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.ID, Token: "backup-held", - Kind: kindBackupHeld, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, - Machine: machines[0], Also: machines[1:], - Summary: fmt.Sprintf("the delivery %s is held: %d of its backups failed or did not answer, so nothing is "+ - "asked or sent", p.ID, len(said)), - Said: strings.Join(said, "; ")}) - } - return out -} - -// probeBackupHeld reads the open plans for the ones held for their backups. -func probeBackupHeld(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - plans, err := d.open.inventory.OpenPlans(ctx) - if err != nil { - return nil, err - } - return observeBackupHolds(plans), nil -} diff --git a/cmd/mesh-controller/plan_backups_hold_test.go b/cmd/mesh-controller/plan_backups_hold_test.go deleted file mode 100644 index 6294e204..00000000 --- a/cmd/mesh-controller/plan_backups_hold_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A backup that failed or stayed silent holds the plan, is said, and a verb asks it again (novox/hq ADR 0404 -// decision 4, issue 1066). - -// aHeldPlan is a plan whose backup on ace failed and whose backup on g14 is done. -func aHeldPlan(t *testing.T) (*inventory.Plan, backupFacts) { - t.Helper() - entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace", "g14")) - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - p := aPlanOf("mail") - stepBackups(t.Context(), p, facts, backupT0) - seat.nights["ace/mail"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"} - seat.nights["g14/mail"] = &night{OK: true, At: backupT0.Add(time.Minute)} - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - return p, facts -} - -// The held plan says which machine, which module and the backup's own words in its note, and is not called tried again. -func TestAHeldPlanSaysTheMachineTheModuleAndTheWordsInItsNote(t *testing.T) { - p, _ := aHeldPlan(t) - for _, want := range []string{"ace", "mail", "no space left on device", "delivery-backups-again"} { - if !strings.Contains(p.Note, want) { - t.Fatalf("note %q lacks %q", p.Note, want) - } - } - if strings.Contains(p.Note, "tried again") || p.State != inventory.PlanBuilding { - t.Fatalf("a held plan is not failed and not retried: %q %s", p.Note, p.State) - } - if !waitingOnBackups(*p) { - t.Fatal("a plan held by its backups is no tier late (S3 leaves it out)") - } -} - -// delivery-backups-again clears the failed records, and the next step asks those backups again and only those. -func TestBackupsAgainClearsTheFailedRecordsAndTheNextStepAsksThemAgain(t *testing.T) { - p, facts := aHeldPlan(t) - again, err := askBackupsAgain(p, "mesh-delivery", "a person released it") - if err != nil || len(again) != 1 || again[0].Machine != "ace" || again[0].Module != "mail" { - t.Fatalf("again=%+v err=%v", again, err) - } - if p.Backups.State != inventory.BackupPending { - t.Fatalf("state %q", p.Backups.State) - } - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - now := backupT0.Add(10 * time.Minute) - if proceed, err := stepBackups(t.Context(), p, facts, now); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - if strings.Join(seat.asked, " ") != "ace/mail" { - t.Fatalf("asked %v; only the failed backup is asked again", seat.asked) - } - seat.nights["ace/mail"] = &night{OK: true, At: now.Add(time.Minute)} - if proceed, _ := stepBackups(t.Context(), p, facts, now.Add(2*time.Minute)); !proceed { - t.Fatal("the plan must go on once the backup it asked again is done") - } - if strings.Contains(p.Note, "delivery-backups-again") { - t.Fatalf("the hold's note stays after it ended: %q", p.Note) - } -} - -// Refused for a plan with no failed backup. -func TestBackupsAgainIsRefusedForAPlanWithNoFailedBackup(t *testing.T) { - if _, err := askBackupsAgain(aPlanOf("mail"), "x", "y"); err == nil { - t.Fatal("a plan with no backups was asked again") - } - p := aPlanOf("mail") - p.Backups = &inventory.PlanBackups{State: inventory.BackupPending, - Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}} - if _, err := askBackupsAgain(p, "x", "y"); err == nil { - t.Fatal("a plan whose backups are pending was asked again") - } -} - -// The verb's argument vector: the plan and a reason are required. -func TestTheVerbDeliveryBackupsAgainTakesAPlanAndAWhy(t *testing.T) { - got, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"}) - want := []string{"delivery", "backups-again", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"} - if err != nil || strings.Join(got, "|") != strings.Join(want, "|") { - t.Fatalf("got %v %v, want %v", got, err, want) - } - if _, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1"}); err == nil { - t.Fatal("no reason was accepted") - } -} - -// One condition per held plan, the backup one, with the words in its evidence; none for a plan only waiting. -func TestAHeldPlanRaisesOneConditionWithTheBackupsWords(t *testing.T) { - held, _ := aHeldPlan(t) - waiting := aPlanOf("mail") - waiting.ID = "plan-w" - waiting.Backups = &inventory.PlanBackups{State: inventory.BackupPending, - Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}} - got := observeBackupHolds([]inventory.Plan{*held, *waiting}) - if len(got) != 1 { - t.Fatalf("observations %+v", got) - } - o := got[0] - if o.Kind != kindBackupHeld || o.Scope != conditions.ScopePlan || o.ID != held.ID || - !strings.Contains(o.Said, "no space left on device") || !strings.Contains(o.Said, "mail") || !strings.Contains(o.Said, "ace") { - t.Fatalf("%+v", o) - } -} - -// A holder off the bus cannot be asked: the plan waits within the bound, retried by the look and not by the error -// path, and at the bound the backup is failed with the reason, so the hold and the condition take over. -func TestABackupHolderThatCannotBeAskedIsFailedAtTheBoundWithTheReason(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - was := askBackupNow - t.Cleanup(func() { askBackupNow = was }) - calls := 0 - askBackupNow = func(_ context.Context, machine, module string) error { - calls++ - return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: no responders", machine)} - } - entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace")) - p := aPlanOf("mail") - if proceed, err := stepBackups(t.Context(), p, facts, backupT0); proceed || err != nil || p.Backups.State != inventory.BackupPending { - t.Fatalf("proceed=%v err=%v %+v", proceed, err, p.Backups) - } - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin/2)); proceed || err != nil || - p.Backups.State != inventory.BackupPending || calls != 2 || p.Backups.Asked[0].AskedAt != nil { - t.Fatalf("within the bound: proceed=%v err=%v calls=%d %+v", proceed, err, calls, p.Backups) - } - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin)); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - a := p.Backups.Asked[0] - if p.Backups.State != inventory.BackupFailed || a.State != inventory.BackupFailed || - !strings.Contains(a.Words, "could not be asked: ") || !strings.Contains(a.Words, "no responders") { - t.Fatalf("%+v", p.Backups) - } - if strings.Contains(p.Note, "tried again") || !strings.Contains(p.Note, "held") { - t.Fatalf("note %q", p.Note) - } -} diff --git a/cmd/mesh-controller/plan_backups_test.go b/cmd/mesh-controller/plan_backups_test.go deleted file mode 100644 index 644f1950..00000000 --- a/cmd/mesh-controller/plan_backups_test.go +++ /dev/null @@ -1,276 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The backups before a delivery plan asks anything (novox/hq ADR 0404 decisions 2 and 3, issue 1065). - -// aModuleWith is a catalogue entry for a module keeping the given data items, on the given machines. -func aModuleWith(t *testing.T, name string, on []string, items ...string) inventory.Entry { - t.Helper() - raw := `{"module":"` + name + `","version":"1"` - if len(items) > 0 { - raw += `,"data":{"own":[` + strings.Join(items, ",") + `]}` - } - raw += `}` - var m catalogue.Manifest - if err := json.Unmarshal([]byte(raw), &m); err != nil { - t.Fatal(err) - } - return inventory.Entry{Manifest: m, On: on} -} - -func anItem(id, extra string) string { - if extra != "" { - extra = "," + extra - } - return `{"id":"` + id + `","path":"/x/` + id + `","class":"valuable","why":"w"` + extra + `}` -} - -func holders(nodes ...string) []catalogue.Held { - var out []catalogue.Held - for _, n := range nodes { - out = append(out, catalogue.Held{Claim: catalogue.BackupSeat, Node: n, Module: "restic"}) - } - return out -} - -// aFakeBackupSeat stands in for the machines' node-backup holders. -type aFakeBackupSeat struct { - asked []string // "machine/module", in order - nights map[string]*night - refuse map[string]string -} - -func (f *aFakeBackupSeat) install(t *testing.T) { - t.Helper() - was, wasRead := askBackupNow, readLastNights - t.Cleanup(func() { askBackupNow, readLastNights = was, wasRead }) - askBackupNow = func(_ context.Context, machine, module string) error { - f.asked = append(f.asked, machine+"/"+module) - if why := f.refuse[machine+"/"+module]; why != "" { - return fmt.Errorf("%s", why) - } - return nil - } - readLastNights = func(_ context.Context, machine string) (map[string]*night, error) { - out := map[string]*night{} - for k, n := range f.nights { - if m, mod, _ := strings.Cut(k, "/"); m == machine { - out[mod] = n - } - } - return out, nil - } -} - -func aPlanOf(modules ...string) *inventory.Plan { - p := &inventory.Plan{ID: "plan-b", State: inventory.PlanBuilding, Tiers: [][]string{modules}, - Modules: map[string]*inventory.PlanModule{}} - for _, m := range modules { - p.Modules[m] = &inventory.PlanModule{} - } - return p -} - -var backupT0 = time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - -// Only a module with an item protected by a backup is asked, and on the machines that run it. -func TestOnlyModulesWithAnItemProtectedByABackupAreAsked(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - entries := []inventory.Entry{ - aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", "")), - aModuleWith(t, "dumped", []string{"ace"}, anItem("b", `"backup":{"dump":"pg_dump","into":"d"}`), anItem("d", "")), - aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)), - aModuleWith(t, "unbacked", []string{"ace"}, anItem("e", `"backup":"none"`)), - aModuleWith(t, "cached", []string{"ace"}, `{"id":"f","path":"/x/f","class":"cache"}`), - aModuleWith(t, "dataless", []string{"ace"}), - } - p := aPlanOf("copied", "dumped", "mirrored", "unbacked", "cached", "dataless") - facts := backupFactsFrom(entries, holders("ace", "g14")) - proceed, err := stepBackups(t.Context(), p, facts, backupT0) - if err != nil || proceed { - t.Fatalf("proceed=%v err=%v; the plan must wait for the backups it asked", proceed, err) - } - want := "ace/copied ace/dumped g14/copied" - if got := strings.Join(seat.asked, " "); got != want { - t.Fatalf("asked %q, want %q", got, want) - } - if p.Backups == nil || p.Backups.State != inventory.BackupPending || len(p.Backups.Asked) != 3 { - t.Fatalf("the plan's record: %+v", p.Backups) - } -} - -// A machine with no backup holder is asked nothing, the plan says so, and it does not hold the plan. -func TestANodeWithNoBackupHolderAsksNothingAndThePlanSaysSo(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "bare"}, anItem("a", ""))} - p := aPlanOf("copied") - proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0) - if err != nil || proceed { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - if strings.Join(seat.asked, " ") != "ace/copied" { - t.Fatalf("asked %v", seat.asked) - } - if p.Backups == nil || len(p.Backups.NoHolder) != 1 || p.Backups.NoHolder[0] != "bare" { - t.Fatalf("the plan must say bare has no holder: %+v", p.Backups) - } - // Only the no-holder machine: nothing to wait for, the plan goes on, and still says it. - only := []inventory.Entry{aModuleWith(t, "copied", []string{"bare"}, anItem("a", ""))} - q := aPlanOf("copied") - proceed, err = stepBackups(t.Context(), q, backupFactsFrom(only, holders("ace")), backupT0) - if err != nil || !proceed || q.Backups == nil || q.Backups.NoHolder[0] != "bare" || q.Backups.State != inventory.BackupDone { - t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, q.Backups) - } -} - -// Nothing of the plan goes on until every backup answered done; a backup is not asked twice. -func TestNothingIsAskedOfTheBuildOrSentUntilEveryBackupAnsweredDone(t *testing.T) { - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace", "g14")) - p := aPlanOf("copied") - if proceed, _ := stepBackups(t.Context(), p, facts, backupT0); proceed { - t.Fatal("went on before any backup answered") - } - // ace is done, g14 still running (its last night is from before the ask). - seat.nights["ace/copied"] = &night{OK: true, At: backupT0.Add(time.Minute)} - seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(-time.Hour)} - if proceed, _ := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed { - t.Fatal("went on while g14's backup had not answered") - } - seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(3 * time.Minute)} - proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(4*time.Minute)) - if err != nil || !proceed || p.Backups.State != inventory.BackupDone { - t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, p.Backups) - } - if len(seat.asked) != 2 { - t.Fatalf("asked %v; each backup is asked once", seat.asked) - } -} - -// A failed backup, and a silent one past the bound, are recorded with the backup's own words and the plan stops. -func TestAFailedOrSilentBackupIsRecordedAndThePlanStopsAsking(t *testing.T) { - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace", "g14")) - p := aPlanOf("copied") - stepBackups(t.Context(), p, facts, backupT0) - seat.nights["ace/copied"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"} - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin+time.Minute)); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - got := map[string]inventory.PlanBackup{} - for _, b := range p.Backups.Asked { - got[b.Machine] = b - } - if got["ace"].State != inventory.BackupFailed || got["ace"].Words != "no space left on device" { - t.Fatalf("ace: %+v", got["ace"]) - } - if got["g14"].State != inventory.BackupFailed || !strings.Contains(got["g14"].Words, "did not answer in ten minutes") { - t.Fatalf("g14: %+v", got["g14"]) - } - if p.Backups.State != inventory.BackupFailed { - t.Fatalf("state %q", p.Backups.State) - } - // It stays stopped, asks nothing more, and says no retry in the plan's note. - n := len(seat.asked) - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(time.Hour)); proceed || err != nil || len(seat.asked) != n { - t.Fatalf("a failed backup must stay stopped: proceed=%v err=%v asked=%v", proceed, err, seat.asked) - } -} - -// A holder that refuses the ask says why, in the record. -func TestAHolderThatRefusesTheAskIsRecordedFailedWithItsWords(t *testing.T) { - seat := &aFakeBackupSeat{refuse: map[string]string{"ace/copied": "copied declares nothing to back up"}} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace"}, anItem("a", ""))} - p := aPlanOf("copied") - if proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - if b := p.Backups.Asked[0]; b.State != inventory.BackupFailed || b.Words != "copied declares nothing to back up" { - t.Fatalf("%+v", b) - } -} - -// A plan moving no backed-up item asks nothing and keeps no record (design 47: costs nothing). -func TestAPlanThatMovesNoBackedUpItemAsksNothing(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - entries := []inventory.Entry{ - aModuleWith(t, "dataless", []string{"ace"}), - aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)), - } - p := aPlanOf("dataless", "mirrored") - proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0) - if err != nil || !proceed || p.Backups != nil || len(seat.asked) != 0 { - t.Fatalf("proceed=%v err=%v record=%+v asked=%v", proceed, err, p.Backups, seat.asked) - } -} - -// The holders of a node seat are every machine's, and a source that fails is no machine without a holder. -func TestTheBackupHoldersAreEveryMachinesAndAFailedReadIsNotNoHolder(t *testing.T) { - entries := func(context.Context) ([]inventory.Entry, error) { - return []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))}, nil - } - f, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return holders("ace", "g14"), nil }, t.Context()) - if err != nil || !f.holder["ace"] || !f.holder["g14"] { - t.Fatalf("%+v %v", f, err) - } - if _, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return nil, fmt.Errorf("store down") }, - t.Context()); err == nil || !strings.Contains(err.Error(), "store down") { - t.Fatalf("a failed read of the holders must fail the step: %v", err) - } -} - -// Issue 1086: the cut of a plan asked its first tier itself, so gitea was sent to novox with no backup asked. -func giteaOnNovox(t *testing.T) inventory.Entry { - return aModuleWith(t, "gitea", []string{"novox"}, - `{"id":"data","path":"/var/lib/gitea/data","class":"valuable","why":"repositories"}`) -} - -func TestACutPlanAsksTheBackupOfGiteaOnNovoxBeforeItAsksItsFirstTier(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - facts := backupFactsFrom([]inventory.Entry{giteaOnNovox(t)}, holders("novox", "ace")) - p := aPlanOf("gitea", "mesh-delivery") - tierAsked := false - err := askFirstTierAtCut(p, - func(p *inventory.Plan) (bool, error) { return stepBackups(t.Context(), p, facts, backupT0) }, - func() error { tierAsked = true; return nil }) - if err != nil || tierAsked { - t.Fatalf("the first tier was asked before the backup: asked=%v err=%v", tierAsked, err) - } - if strings.Join(seat.asked, " ") != "novox/gitea" || p.Backups == nil || p.Backups.State != inventory.BackupPending { - t.Fatalf("asked %v, record %+v", seat.asked, p.Backups) - } - if !strings.HasPrefix(p.Note, backupsNote) { - t.Fatalf("note %q", p.Note) - } -} - -func TestACutPlanWhoseBackupsCouldNotBeWorkedOutAsksNothingAndSaysSo(t *testing.T) { - p := aPlanOf("gitea") - tierAsked := false - err := askFirstTierAtCut(p, - func(*inventory.Plan) (bool, error) { return false, fmt.Errorf("the catalogue could not be read") }, - func() error { tierAsked = true; return nil }) - if err != nil || tierAsked || !strings.Contains(p.Note, "could not be worked out") { - t.Fatalf("asked=%v err=%v note %q", tierAsked, err, p.Note) - } -} diff --git a/cmd/mesh-controller/plan_clock_integration_test.go b/cmd/mesh-controller/plan_clock_integration_test.go deleted file mode 100644 index 51e968d8..00000000 --- a/cmd/mesh-controller/plan_clock_integration_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - // app's key is not built, so the plan stands on its tier as it did while the build ran; with the stand-in builder's - // key done at once, one step would carry it to done. - holdKeys(t, "app") - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 0 { - t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" || - !before.Modules["app"].Key || before.Modules["app"].KeyAt != "c1aaaaaaaa" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } - if len(*asked) != 0 { - t.Fatalf("advancing took the key in: %v", *asked) - } -} diff --git a/cmd/mesh-controller/plan_clock_test.go b/cmd/mesh-controller/plan_clock_test.go deleted file mode 100644 index 7de204d1..00000000 --- a/cmd/mesh-controller/plan_clock_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 296: a plan saved a few seconds ago that has stood in its tier for forty minutes says -// forty minutes, and LATE — the line counted from the last save, which every advance made. -func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) { - now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC) - asked := now.Add(-40 * time.Minute) - p := inventory.Plan{ID: "plan-t", Repository: "novox/mesh-catalog", Commit: "3ad22356", State: inventory.PlanBuilding, - Created: now.Add(-40 * time.Minute), Updated: now.Add(-3 * time.Second), TierEntered: now.Add(-40 * time.Minute), - Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}} - - line := planLine(p, now) - if !strings.Contains(line, "building for 40m0s") || !strings.Contains(line, "LATE") { - t.Fatalf("a plan in its tier for 40m, saved 3s ago, reads %q", line) - } - st := planStatuses([]inventory.Plan{p}, now, nil)[0] - if !st.Late || !st.Since.Equal(p.TierEntered) { - t.Errorf("status --json says %+v", st) - } - if _, late := openPlans([]inventory.Plan{p}, now, nil); late != 1 { - t.Errorf("status counted %d late", late) - } - - // LATE is the stalled condition said on the line: the same start and the same bound, measured or not. - bounds := boundsOf([]inventory.Duration{{Subject: p.Repository, Took: 20 * time.Minute}}) - if got := bounds.of(p.Repository); got != time.Hour { - t.Fatalf("a repository whose tiers took 20m is given %s", got) - } - if got := bounds.of("novox/other"); got != tierAtLeast { - t.Fatalf("a repository with nothing measured is given %s", got) - } - for _, in := range []time.Duration{40 * time.Minute, 70 * time.Minute} { - q := p - q.TierEntered = now.Add(-in) - bound := bounds.of(q.Repository) - late := strings.Contains(planLineWith(q, now, bound), "LATE") - stalled := len(watchPlans(&signalFacts{now: now, plans: []planFacts{{id: q.ID, repository: q.Repository, - commit: q.Commit, tiers: 1, entered: inTierSince(q), bound: bound}}})) == 1 - if late != stalled || late != (in > bound) { - t.Errorf("in its tier %s with a bound of %s: LATE %v, stalled %v", in, bound, late, stalled) - } - } - - // A plan read without its tier's stamp counts from its last save, the only time it has. - old := p - old.TierEntered = time.Time{} - if line := planLine(old, now); !strings.Contains(line, "for 3s") || strings.Contains(line, "LATE") { - t.Errorf("a plan without its tier's stamp reads %q", line) - } - - // A walk that waited for its delivery's word counts its wait from when it was opened, and its tier - // from the word. - waiting := p - waiting.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat} - waiting.Modules = map[string]*inventory.PlanModule{} - if line := planLine(waiting, now); !strings.Contains(line, "for 40m0s") || strings.Contains(line, "LATE") { - t.Errorf("a walk waiting 40m reads %q", line) - } - if st := planStatuses([]inventory.Plan{waiting}, now, nil)[0]; st.Late { - t.Errorf("a waiting walk is called late: %+v", st) - } - let := now.Add(-5 * time.Minute) - gone := p - gone.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat, Go: &let} - if line := planLine(gone, now); !strings.Contains(line, "building for 5m0s") || strings.Contains(line, "LATE") { - t.Errorf("a walk let go 5m ago, opened 40m ago, reads %q", line) - } -} - -// novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision, -// its last save and its `plan-moved` stay where its last change left them. - -// novox/hq issue 920: a building plan whose current tier is all built waits on its note (a send refused at the -// gate), and its line, `status` and the stalled condition say that, not "building for ... LATE". -func TestAPlanLineSaysTheRefusalOnceItsTierIsBuilt(t *testing.T) { - now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) - built := now.Add(-time.Hour) - refusal := "tier 1: a build waits there for its gate: novox (mesh-delivery-indexer d68d7ef3→a5aa69a4) — tried again" - mk := func(second string) inventory.Plan { - return inventory.Plan{ID: "plan-r", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding, - Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: refusal, - Tiers: [][]string{{"a"}, {"b", "c"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, - "b": {State: "built", BuiltAt: &built}, "c": {State: second}}} - } - - // 920/AC1 - p := mk("built") - line := planLine(p, now) - if !strings.Contains(line, "a build waits there for its gate: novox") || strings.Contains(line, "building for") { - t.Errorf("920/AC1: a plan with its tier built and a refusal reads %q", line) - } - st := planStatuses([]inventory.Plan{p}, now, nil) - if len(st) != 1 || !st[0].Late { - t.Errorf("920/AC1: status: %+v", st) - } - - // 920/AC2 - if line := planLine(mk("asked"), now); !strings.Contains(line, "building for 1h0m0s") { - t.Errorf("920/AC2: a plan with a tier module not built reads %q", line) - } -} - -// 920/AC3: a tier move writes the previous tier's summary into the note; with the new tier all built, the line still -// says "building for", for that note is no refusal. -func TestAPlanLineDoesNotHoldOnATiersSummary(t *testing.T) { - now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) - built := now.Add(-time.Hour) - for _, note := range []string{"failed or refused its send (apply-failing says it): novox", "tier 0 built; sent mesh-sdk to novox first, in one send"} { - p := inventory.Plan{ID: "plan-s", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding, - Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: note, - Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, "b": {State: "built", BuiltAt: &built}}} - if line := planLine(p, now); !strings.Contains(line, "building for 1h0m0s") || strings.Contains(line, "held") { - t.Errorf("920/AC3: a plan with the note %q reads %q", note, line) - } - } -} diff --git a/cmd/mesh-controller/plan_failed.go b/cmd/mesh-controller/plan_failed.go deleted file mode 100644 index 2e3011ec..00000000 --- a/cmd/mesh-controller/plan_failed.go +++ /dev/null @@ -1,227 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "regexp" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// failedModule is one module of a failed plan that failed to build, where it runs and what it said. -type failedModule struct { - module string - machines []string - why string -} - -// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1). -type failedPlanFacts struct { - id, repository, commit string - modules []string - failed []failedModule - // note is where the plan stopped when no module failed to build (a rollout stopped at its first machine). - note string - // since is the plan's last change: when it failed. - since time.Time - // refusal is why `plans retry` would not run it, empty when it would. - refusal string - // passed is set when a newer plan of its repository and branch is done and the retry is refused for that - // (novox/hq ADR 0451): the condition asks nothing. - passed bool -} - -// machines are the machines the failed modules run on, sorted, each once. -func (p failedPlanFacts) machines() []string { - seen := map[string]bool{} - var out []string - for _, m := range p.failed { - for _, n := range m.machines { - if !seen[n] { - seen[n] = true - out = append(out, n) - } - } - } - sort.Strings(out) - return out -} - -// failureSaid is what failed, in the mesh's own words, for the condition's evidence. -func (p failedPlanFacts) failureSaid() string { - var parts []string - for _, m := range p.failed { - line := m.module - if len(m.machines) > 0 { - line += " on " + strings.Join(m.machines, ", ") - } - if m.why != "" { - line += ": " + oneLine(m.why) - } - parts = append(parts, line) - } - if len(parts) == 0 { - return "stopped: " + oneLine(p.note) - } - return strings.Join(parts, "; ") -} - -// refusalSaid is why a retry would not run, for the evidence. -func (p failedPlanFacts) refusalSaid() string { - if p.refusal == "" { - return "" - } - return "; a retry would not run: " + oneLine(p.refusal) -} - -// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan, -// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs). -func retryAction(plan string) conditions.Action { - return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, - Arguments: map[string]string{"retry": plan}} -} - -var ( - planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`) - walkWd = regexp.MustCompile(`\bwalks\b`) - walkOne = regexp.MustCompile(`\bwalk\b`) -) - -// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a -// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the -// words say only that a retry would not run. -func plainRefusal(refusal string) string { - return plainRefusalOr(refusal, "a retry would not run now; the details say why") -} - -// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly. -func plainRefusalOr(refusal, fallback string) string { - r := planIDWord.ReplaceAllString(refusal, "the plan") - r = walkWd.ReplaceAllString(r, "deliveries") - r = walkOne.ReplaceAllString(r, "delivery") - r = strings.TrimSpace(r) - if _, ok := conditions.Plain(r); !ok { - return fallback - } - return r -} - -// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that -// is plain, and Retry when a retry would run, else the refusal's words. -func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) { - name := deliveryName(p.modules, p.repository) - headline = conditions.Capital(name) + " failed" - what := deliveryWhat(p.modules, p.repository) - var parts []string - for i, m := range p.failed { - if i == 2 { - parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2)) - break - } - line := m.module - if len(m.machines) > 0 { - line += " for " + readableList(m.machines) - } - if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" { - if _, ok := conditions.Plain(w); ok { - line += " (" + w + ")" - } - } - parts = append(parts, line) - } - if len(parts) > 0 { - explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts)) - } else { - explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what) - } - switch { - case p.passed: - explanation += " A newer delivery has passed it, so a retry would send older code. Nothing is needed from you." - case p.refusal == "": - explanation += " Nothing is tried again until you say so." - needs = "try the failed builds again " + FromMeshMCPServer - actions = []conditions.Action{retryAction(p.id)} - default: - explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "." - needs = "look at why it failed; a retry would not run now." - } - resolved = conditions.Capital(name) + " is no longer failed" - return -} - -// cutWords is s cut to at most n bytes at a word. -func cutWords(s string, n int) string { - if len(s) <= n { - return s - } - t := s[:n] - if i := strings.LastIndexByte(t, ' '); i > n/2 { - t = t[:i] - } - return strings.TrimRight(t, " ,;:") + "…" -} - -// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator -// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not -// stopped by a person. Each carries why `plans retry` would not run it, when it would not. -func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) { - plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin)) - if err != nil { - return nil, fmt.Errorf("the plans cannot be read: %w", err) - } - var failed []inventory.Plan - for _, p := range plans { - if failedAndAsked(p) { - failed = append(failed, p) - } - } - if len(failed) == 0 { - return nil, nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil, fmt.Errorf("the catalogue cannot be read: %w", err) - } - onPath := ownPathOf(entries) - runsOn := map[string][]string{} - for _, e := range entries { - runsOn[e.Manifest.Module] = e.On - } - var out []failedPlanFacts - for _, p := range failed { - out = append(out, failedFactsOf(p, plans, onPath, runsOn)) - } - return out, nil -} - -// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not -// stopped by a person. -func failedAndAsked(p inventory.Plan) bool { - return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == "" -} - -// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each -// module runs. -func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string, - runsOn map[string][]string) failedPlanFacts { - f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p), - note: p.Note, since: p.Updated} - for _, m := range f.modules { - if s := p.Modules[m]; s != nil && s.State == "failed" { - machines := append([]string(nil), runsOn[m]...) - sort.Strings(machines) - f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why}) - } - } - if err := retryRefusal(p, plans, onPath); err != nil { - f.refusal = err.Error() - var passed passedBy - f.passed = errors.As(err, &passed) - } - return f -} diff --git a/cmd/mesh-controller/plan_failed_test.go b/cmd/mesh-controller/plan_failed_test.go deleted file mode 100644 index 05691840..00000000 --- a/cmd/mesh-controller/plan_failed_test.go +++ /dev/null @@ -1,338 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314). - -func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan { - p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed, - Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC), - Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}}, - Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} - if mutate != nil { - mutate(&p) - } - return p -} - -func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts { - return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{}, - map[string][]string{"openrazer": {"g14", "shanks"}}) -} - -// 1314/AC1 -func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - p := aFailedPlan("plan-1792433168676870614", nil) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(p)} - got := watchPlansFailed(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - o := got[0] - if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator || - o.Scope != conditions.ScopePlan || o.ID != p.ID { - t.Fatalf("kind, severity, resolver or subject: %+v", o) - } - if len(o.Actions) != 1 { - t.Fatalf("actions: %+v", o.Actions) - } - a := o.Actions[0] - if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove || - len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID { - t.Fatalf("the action is not Retry with only the plan: %+v", a) - } - if o.Needs == "" { - t.Fatal("no Needs: the asker wants an action with a need") - } - for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} { - if !strings.Contains(o.Headline+o.Explanation+o.Said, want) { - t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said) - } - } - if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") { - t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation) - } - words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions} - if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { - t.Errorf("the words are not plain: %s\n%+v", why, words) - } -} - -// 1314/AC2 -func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) { - for name, p := range map[string]inventory.Plan{ - "held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }), - "stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }), - "no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }), - "building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }), - } { - if failedAndAsked(p) { - t.Errorf("%s: a plan not asked of the operator is asked", name) - } - } - if !failedAndAsked(aFailedPlan("plan-5", nil)) { - t.Error("a failed delivery plan is not asked") - } -} - -// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it. -func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) }) - newer := aFailedPlan("plan-2", func(p *inventory.Plan) { - p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute) - }) - superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" }) - done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone }) - first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC) - gate := aFailedPlan("plan-5", func(p *inventory.Plan) { - p.Note = "openrazer stopped at its first machine" - p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"}, - Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"}, - Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}} - }) - for name, c := range map[string]struct { - p inventory.Plan - others []inventory.Plan - says string - }{ - "a newer plan": {older, []inventory.Plan{newer}, "supersedes it"}, - "superseded": {superseded, nil, "superseded"}, - "done": {done, nil, "nothing to retry"}, - "gate failed": {gate, nil, "failed its gate"}, - } { - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)} - got := watchPlansFailed(f) - if len(got) != 1 { - t.Fatalf("%s: %+v", name, got) - } - o := got[0] - if len(o.Actions) != 0 { - t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions) - } - if !strings.Contains(o.Said, c.says) { - t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said) - } - if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) { - t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation) - } - words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} - if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { - t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words) - } - } -} - -// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23. -func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - defer k.Close(t.Context()) - p := aFailedPlan("plan-1792433168676870614", nil) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(p)} - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - open, _ := k.Open(t.Context()) - if len(open) != 1 || open[0].Kind != "plan-failed" { - t.Fatalf("not raised: %+v", open) - } - // Retried: the plan is building, so it is no longer among the failed. - f = calm(now.Add(time.Minute)) - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - if open, _ = k.Open(t.Context()); len(open) != 0 { - t.Fatalf("the condition outlived the failure: %+v", open) - } - // Failed again: asked again, with Retry, and nothing retried it. - f = calm(now.Add(2 * time.Minute)) - again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) }) - f.failed = []failedPlanFacts{failedFactsFor(again)} - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - open, _ = k.Open(t.Context()) - if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" { - t.Fatalf("a second failure was not asked again with Retry: %+v", open) - } -} - -// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing -// S23 produces is an observation that offers an answer. -func TestNoClockRetriesAFailedPlan(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))} - for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} { - f.now = now.Add(later) - got := watchPlansFailed(f) - if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" { - t.Fatalf("after %s: %+v", later, got) - } - } - // A plan failed longer ago than the week it is kept is left to `plans`. - f.now = now.Add(planFailedWithin + time.Second) - if got := watchPlansFailed(f); len(got) != 0 { - t.Fatalf("a plan failed beyond the window is still raised: %+v", got) - } -} - -// A later plan of the same repository and branch, in a state, made after the failed one. -func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan { - q := aFailedPlan(id, func(p *inventory.Plan) { - p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour) - }) - return q -} - -// 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan. -func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) { - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) - later := laterPlan("plan-2", "main", inventory.PlanDone, failed) - err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{}) - if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") { - t.Fatalf("the retry was not refused, naming the later plan: %v", err) - } -} - -// 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing. -func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) { - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) - open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed) - err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{}) - if err == nil || !strings.Contains(err.Error(), "supersedes it") { - t.Fatalf("with the later plan open the refusal is not today's: %v", err) - } - other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed) - if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil { - t.Fatalf("a later done plan of another branch refused the retry: %v", err) - } - earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed) - earlier.Created = failed.Created.Add(-time.Hour) - if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil { - t.Fatalf("an earlier done plan refused the retry: %v", err) - } -} - -// 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery. -func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) }) - later := laterPlan("plan-2", "main", inventory.PlanDone, failed) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(failed, later)} - got := watchPlansFailed(f) - if len(got) != 1 || len(got[0].Actions) != 0 { - t.Fatalf("%+v", got) - } - o := got[0] - if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") { - t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation) - } - if strings.Contains(o.Explanation, "details say why") { - t.Errorf("the generic fallback was said: %q", o.Explanation) - } - words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} - if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { - t.Errorf("not plain: %s\n%+v", why, words) - } -} - -// plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since. -type plansBehind []inventory.Plan - -func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) { - var out []inventory.Plan - for _, p := range s { - if p.Open() { - out = append(out, p) - } - } - return out, nil -} - -func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) { - all := append([]inventory.Plan(nil), s...) - sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) }) - if len(all) > limit { - all = all[:limit] - } - return all, nil -} - -func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) { - var out []inventory.Plan - for _, p := range s { - if p.Created.After(since) || p.Open() { - out = append(out, p) - } - } - return out, nil -} - -// 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it. -func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) { - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) }) - all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)} - for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50 - other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed) - other.Repository = "novox/other" - other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour) - all = append(all, other) - } - plans, err := plansForRetry(t.Context(), all, failed) - if err != nil { - t.Fatal(err) - } - if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") { - t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err) - } -} - -// ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its -// plan's last change passes 7 days. -func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) }) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(old)} - if got := watchPlansFailed(f); len(got) != 0 { - t.Fatalf("a plan last changed 8 days ago raised %+v", got) - } - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - defer k.Close(t.Context()) - fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) }) - f.failed = []failedPlanFacts{failedFactsFor(fresh)} - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - if open, _ := k.Open(t.Context()); len(open) != 1 { - t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open) - } - f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - if open, _ := k.Open(t.Context()); len(open) != 0 { - t.Fatalf("the condition outlived the 7 days: %+v", open) - } -} diff --git a/cmd/mesh-controller/plan_ids.go b/cmd/mesh-controller/plan_ids.go deleted file mode 100644 index d9dca230..00000000 --- a/cmd/mesh-controller/plan_ids.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "fmt" - "regexp" -) - -// planID is every id the mesh gives a plan: a merge's walk or batch (plan-) or a release -// (release-). -var planID = regexp.MustCompile(`^(plan|release)-[0-9]+$`) - -// planIDArguments are the arguments of plans that name a plan. wait is among them for when it is declared -// (novox/hq issue 229); an argument not declared is refused before this is asked. -var planIDArguments = []string{"id", "wait", "stop", "close", "retry", "go"} - -// plainPlanIDs refuses a plans call naming a plan by anything but a plan's id (novox/hq issue 561): checked -// once, where the call becomes a command line, so nothing else a caller sent reaches a command or the store. -func plainPlanIDs(given map[string]string) error { - for _, arg := range planIDArguments { - if id, named := given[arg]; named && !planID.MatchString(id) { - return fmt.Errorf("plans %s: %q is not a plan's id (plan- or release-, as `plans` "+ - "lists them); nothing was done", arg, id) - } - } - return nil -} diff --git a/cmd/mesh-controller/plan_ids_test.go b/cmd/mesh-controller/plan_ids_test.go deleted file mode 100644 index f49039f4..00000000 --- a/cmd/mesh-controller/plan_ids_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "strings" - "testing" -) - -// **A plan's id is checked once, where a call becomes a command line** (novox/hq issue 561): every argument of -// plans that names a plan is refused unless it is an id the mesh gives a plan, so a quote never reaches a -// command, nor the store behind it. -func TestAPlanIdThatIsNoPlansIdIsRefusedBeforeACommandStarts(t *testing.T) { - for _, arg := range []string{"id", "stop", "close", "retry", "go"} { - for _, bad := range []string{"x' or '1'='1", "plan-1'", "plan-", "plan-1 2", "batch-1", "PLAN-1"} { - args := map[string]any{arg: bad} - if arg == "stop" || arg == "close" || arg == "go" { - args["why"] = "a test" - } - argv, err := argvFor("plans", args) - if err == nil { - t.Errorf("plans %s %q was composed: %v", arg, bad, argv) - continue - } - if !strings.Contains(err.Error(), "is not a plan's id") { - t.Errorf("plans %s %q was refused for another reason: %v", arg, bad, err) - } - } - for _, good := range []string{"plan-1791813626034291775", "release-1791811938267404408"} { - args := map[string]any{arg: good} - if arg == "stop" || arg == "close" || arg == "go" { - args["why"] = "a test" - } - if _, err := argvFor("plans", args); err != nil { - t.Errorf("plans %s %q was refused: %v", arg, good, err) - } - } - } -} diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go deleted file mode 100644 index f867ad6b..00000000 --- a/cmd/mesh-controller/plan_keys.go +++ /dev/null @@ -1,268 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "sync" - "time" - - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6 -// and A7). -// -// A walk asks no build of anybody (R4g; ADR 0453 §5). A module of its tier is marked -// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value -// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every -// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done -// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words. -// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules). - -// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's). -type planKeyReads interface { - ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error) - Build(ctx context.Context, module, fp string) (builderkeys.Key, error) -} - -// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus. -var openPlanKeys = func() (planKeyReads, func(), error) { - r, closeKeys, err := openBuilderKeys() - if err != nil { - return nil, nil, err - } - return r, closeKeys, nil -} - -// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6). -func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) { - state.BusWaitFrom = nil - state.State = "asked" - state.AskedAt = &now - state.Key = true - state.Build, state.Fingerprint = "", "" - state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref)) - state.Why = "waits for the builder's key" - state.Commit, state.BuiltAt, state.Made = "", nil, "" - fmt.Printf(" tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module) -} - -// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it -// waits: kept in this process, so a look starts each once and a settle says why it waits. -var keyTakes = struct { - sync.Mutex - running map[string]bool - waits map[string]string - // taken are the keys whose take-in ended registered, or found the module already at a later build (superseded). - taken map[string]bool -}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}} - -// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's -// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only -// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look. -func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) { - return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) { - inBackground(func() { takeKeyIn(context.WithoutCancel(ctx), open, module, trunk, id) }) - }) -} - -// takeKeyIn is how a look takes a done key in; a variable so a test of what a walk does around it needs no builder. -var takeKeyIn = takeKeyOffTheHold - -// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's). -type keyRecords interface { - BuildByID(ctx context.Context, id string) (inventory.Build, bool, error) - RefusedOf(ctx context.Context, ids []string) (map[string]string, error) - SourceOf(ctx context.Context, module string) (inventory.Source, error) -} - -// settleKeys is settleFromKeys on the records handed in, starting a take-in with take. -func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string, - take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) { - changed := false - set := func(s *inventory.PlanModule, why string) { - if s.Why != why { - s.Why = why - changed = true - } - } - for _, m := range tier { - s := p.Modules[m] - if s == nil || !s.Key || s.State != "asked" { - continue - } - trunk, err := keys.ModuleAt(ctx, m, s.KeyAt) - if errors.Is(err, builderkeys.ErrNoValue) { - set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt))) - continue - } - if err != nil { - return changed, err - } - if trunk.FP2 == "" { - why := trunk.None2 - if why == "" { - why = "the indexer wrote no fingerprint of the new form" - } - set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why)) - continue - } - if s.Fingerprint != trunk.FP2 { - s.Fingerprint, changed = trunk.FP2, true - } - k, err := keys.Build(ctx, m, trunk.FP2) - if err != nil { - return changed, err - } - switch { - case k.Failed(): - // **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`, - // `version taken:`), and a rebuild of it is a new revision the next walk takes. - s.State, s.Why = "failed", k.Build.Error - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error)) - changed = true - continue - case !k.Done() || len(k.Unreferenced()) > 0: - set(s, "waits: "+k.Says()) - continue - } - id := keyBuildID(k) - refused, err := inv.RefusedOf(ctx, []string{id}) - if err != nil { - return changed, fmt.Errorf("the refused registrations cannot be read: %w", err) - } - for kept, why := range refusalsKeptFor([]string{id}) { - refused[kept] = why - } - if why, ok := refused[id]; ok { - s.State, s.Why, s.Build = "failed", why, id - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why)) - changed = true - continue - } - b, found, err := inv.BuildByID(ctx, id) - if err != nil { - return changed, err - } - if found && b.Worked() { - keyTakes.Lock() - taken := keyTakes.taken[id] - keyTakes.Unlock() - registered := taken - if !registered { - if registered, err = registeredAt(ctx, inv, m, trunk.Commit); err != nil { - return changed, err - } - } - if registered { - at := b.At - s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, trunk.Commit, id - s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision) - s.Why = "" - fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2)) - changed = true - continue - } - } - keyTakes.Lock() - running, waits := keyTakes.running[id], keyTakes.waits[id] - if !running { - keyTakes.running[id] = true - } - keyTakes.Unlock() - if waits != "" { - set(s, firstLine(waits)) - } else { - set(s, "taking in the builder's done key "+short(trunk.FP2)) - } - if !running { - take(m, trunk, id) - } - } - return changed, nil -} - -// registeredAt says the catalogue holds the module registered at the commit the walk takes it at: its key's take-in -// registered it there, whether with the key's artifacts or, unchanged, with the build it stands on (no move). A key -// recorded at an earlier commit (the same fingerprint at an older merge) is taken in again, to register it here. -func registeredAt(ctx context.Context, inv keyRecords, module, commit string) (bool, error) { - src, err := inv.SourceOf(ctx, module) - if errors.Is(err, inventory.ErrNoSuchModule) { - return false, nil - } - if err != nil { - return false, err - } - return sameCommitID(src.BuiltFrom, commit), nil -} - -func sameCommitID(a, b string) bool { - if a == "" || b == "" { - return false - } - if a == b { - return true - } - n := min(len(a), len(b)) - return n >= 7 && a[:n] == b[:n] -} - -func orNow(commit string) string { - if commit == "" { - return "its value now" - } - return short(commit) -} - -// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the -// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the -// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look. -func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) { - defer func() { - keyTakes.Lock() - delete(keyTakes.running, id) - keyTakes.Unlock() - askToLook() - }() - keys, closeKeys, err := openBuilderKeys() - if err != nil { - keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error()) - return - } - defer closeKeys() - _, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk, - Ask: registrationAsk{Merge: trunk.Commit}}) - switch { - case err == nil, errors.Is(err, inventory.ErrSuperseded): - // Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries - // this merge is answered by, as a build-agent outcome superseded was. - keepKeyWait(id, "") - keyTakes.Lock() - keyTakes.taken[id] = true - keyTakes.Unlock() - case errors.Is(err, errKeyWaits): - keepKeyWait(id, err.Error()) - default: - why := err.Error() - if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil { - fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr) - } - keepRefusalFor(id, why, time.Now()) - } -} - -func keepKeyWait(id, why string) { - keyTakes.Lock() - defer keyTakes.Unlock() - if why == "" { - delete(keyTakes.waits, id) - return - } - keyTakes.waits[id] = why -} diff --git a/cmd/mesh-controller/plan_keys_test.go b/cmd/mesh-controller/plan_keys_test.go deleted file mode 100644 index 3fe15089..00000000 --- a/cmd/mesh-controller/plan_keys_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" - - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" -) - -// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by -// module. -type planKeysOf struct { - values map[string]sdkdelivery.Module // by module + "@" + commit - keys map[string]builderkeys.Key -} - -func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { - if v, ok := k.values[module+"@"+commit]; ok { - return v, nil - } - return sdkdelivery.Module{}, builderkeys.ErrNoValue -} - -func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { - if key, ok := k.keys[module]; ok && key.Fingerprint == fp { - return key, nil - } - return builderkeys.Key{Module: module, Fingerprint: fp}, nil -} - -// recordsOf is the build records and refusals a settle reads, for a test. -type recordsOf struct { - builds map[string]inventory.Build - refused map[string]string - sources map[string]inventory.Source -} - -func (r recordsOf) BuildByID(_ context.Context, id string) (inventory.Build, bool, error) { - b, ok := r.builds[id] - return b, ok, nil -} - -func (r recordsOf) RefusedOf(_ context.Context, ids []string) (map[string]string, error) { - out := map[string]string{} - for _, id := range ids { - if why, ok := r.refused[id]; ok { - out[id] = why - } - } - return out, nil -} - -func (r recordsOf) SourceOf(_ context.Context, module string) (inventory.Source, error) { - if s, ok := r.sources[module]; ok { - return s, nil - } - return inventory.Source{}, inventory.ErrNoSuchModule -} - -func aWalkOfOne(module string) (*inventory.Plan, map[string]inventory.Entry) { - p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{module}}, - Modules: map[string]*inventory.PlanModule{}, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789", - Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789"}}}} - byName := map[string]inventory.Entry{module: {Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + module, Ref: "main"}}} - return p, byName -} - -// A6 (ADR 0503 §6, ADR 0498 check §7): a walk whose module has a done key goes asked to built with no build asked: the -// ask stub is never called, the key is taken in once, and `plans` says it was taken from the builder's key. -func TestAWalkTakesADoneKeyAndAsksNoBuild(t *testing.T) { - p, byName := aWalkOfOne("sudo") - askModule(t.Context(), p, "sudo", byName) - s := p.Modules["sudo"] - if s.State != "asked" || !s.Key || s.Build != "" || s.KeyAt != "c0ffee0123456789" { - t.Fatalf("the module is not taken from a key: %+v", s) - } - fp := "src2:" + strings.Repeat("a", 64) - k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 2, Build: sdkdelivery.BuilderBuild{ - State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", - Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} - keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}}, - keys: map[string]builderkeys.Key{"sudo": k}} - records := recordsOf{builds: map[string]inventory.Build{}, sources: map[string]inventory.Source{}} - var taken []string - take := func(m string, _ sdkdelivery.Module, id string) { taken = append(taken, m+" "+id) } - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - - changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take) - if err != nil || !changed || len(taken) != 1 || s.State != "asked" || s.Fingerprint != fp { - t.Fatalf("a done key not recorded yet is taken in once: %v %v %v %+v", changed, err, taken, s) - } - // A second look while the take-in runs starts no second one. - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || len(taken) != 1 { - t.Fatalf("a second look took the key in again: %v %v", taken, err) - } - // The take-in recorded and registered it: the look settles it built from the record. - id := keyBuildID(k) - records.builds[id] = inventory.Build{ID: id, Module: "sudo", Commit: "c0ffee0123456789", At: time.Now()} - records.sources["sudo"] = inventory.Source{BuiltFrom: "c0ffee0123456789"} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { - t.Fatal(err) - } - if s.State != "built" || s.Build != id || !strings.Contains(s.Made, "taken from the builder's key") { - t.Fatalf("the module is not built from its key: %+v", s) - } -} - -// A7: a key building waits and says so, then moves on when done, with no timer and no ask; a failed key fails the walk -// in the builder's words; a value the indexer has not written yet waits, named. -func TestAWalkWaitsForAKeyAndFailsWithItsWords(t *testing.T) { - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - p, byName := aWalkOfOne("view") - askModule(t.Context(), p, "view", byName) - s := p.Modules["view"] - records := recordsOf{builds: map[string]inventory.Build{}} - take := func(string, sdkdelivery.Module, string) {} - keys := planKeysOf{values: map[string]sdkdelivery.Module{}, keys: map[string]builderkeys.Key{}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || - !strings.Contains(s.Why, "waits for the indexer's value of view at c0ffee01") { - t.Fatalf("no indexer value yet: %q %v", s.Why, err) - } - fp := "src2:" + strings.Repeat("c", 64) - keys.values["view@c0ffee0123456789"] = sdkdelivery.Module{FP2: fp} - keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || - s.State != "asked" || !strings.Contains(s.Why, "building on g14") || p.State != inventory.PlanBuilding { - t.Fatalf("a building key: %+v %v", s, err) - } - keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: the store answered 500"}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { - t.Fatal(err) - } - if s.State != "failed" || p.State != inventory.PlanFailed || !strings.Contains(p.Note, "not stored: tools") { - t.Fatalf("a failed key does not fail the walk in its words: %+v %q", s, p.Note) - } -} - -// A7: a take-in refused (a module from another repository, a manifest refused) fails the walk with the refusal. -func TestARefusedTakeInFailsTheWalk(t *testing.T) { - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - p, byName := aWalkOfOne("sudo") - askModule(t.Context(), p, "sudo", byName) - fp := "src2:" + strings.Repeat("d", 64) - k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{ - State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", - Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} - keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}}, - keys: map[string]builderkeys.Key{"sudo": k}} - records := recordsOf{refused: map[string]string{keyBuildID(k): "sudo is built from novox/mesh-catalog, and this build is of agent/sudo"}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "agent/sudo") { - t.Fatalf("a refused take-in did not fail the walk: %s %q", p.State, p.Note) - } -} - -// A6: a walk's module whose fingerprint did not change since an earlier merge has its key recorded at that merge's -// commit: the walk takes it in again, so the module is registered at the commit the walk carries, before it is built. -func TestAKeyRecordedAtAnEarlierCommitIsTakenInAgain(t *testing.T) { - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - p, byName := aWalkOfOne("sudo") - askModule(t.Context(), p, "sudo", byName) - fp := "src2:" + strings.Repeat("e", 64) - k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{ - State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", - Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} - keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}}, - keys: map[string]builderkeys.Key{"sudo": k}} - id := keyBuildID(k) - records := recordsOf{builds: map[string]inventory.Build{id: {ID: id, Module: "sudo", Commit: "0ld0ld0ld0ld"}}, - sources: map[string]inventory.Source{"sudo": {BuiltFrom: "0ld0ld0ld0ld"}}} - var taken []string - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(m string, _ sdkdelivery.Module, _ string) { - taken = append(taken, m) - }); err != nil { - t.Fatal(err) - } - if s := p.Modules["sudo"]; s.State != "asked" || len(taken) != 1 { - t.Fatalf("a key recorded at an earlier commit settled the walk at that commit: %+v, taken %v", s, taken) - } -} diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go deleted file mode 100644 index d8d84206..00000000 --- a/cmd/mesh-controller/plan_looks.go +++ /dev/null @@ -1,718 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sort" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A rollout advances on events, not on a timer (novox/hq ADR 0406 §5, issue 974). One look at the open plans runs on -// each thing that can let a plan move: a plan saved (a build's `built`, a tier's send kept, a command's move), a -// machine's report on a send, a silent key's expiry, and the controller taking the lease. The looks are asked for -// with askToLook and run one at a time by a worker, so an ask made while a look runs is not lost and many asks are -// one look. Only two things are still started by time, both named and both one-shot: a failed advance step is -// looked at again (retryLook), and the end of a walk whose end is not known is recorded when it can be -// (armWalkEnd). - -// looker runs the look at the plans that events ask for. -type looker struct { - signal chan struct{} // one place: asks made together are one look - look func(ctx context.Context) (failed bool) - // gap is the least time between two looks, so a plan that kept changing could not make the worker spin. - gap time.Duration - // first runs before the first look: what a controller that took the lease sets up. - first func(ctx context.Context) - - mu sync.Mutex - retry *time.Timer - backoff time.Duration - base time.Duration - most time.Duration -} - -const ( - retryLookFirst = 5 * time.Second - retryLookMost = time.Minute - lookGap = 200 * time.Millisecond -) - -func newLooker(look func(ctx context.Context) bool) *looker { - return &looker{signal: make(chan struct{}, 1), look: look, gap: lookGap, base: retryLookFirst, most: retryLookMost} -} - -// ask has a look run, soon; never blocks. -func (l *looker) ask() { - select { - case l.signal <- struct{}{}: - default: - } -} - -// run takes the asks one at a time until the context ends. -func (l *looker) run(ctx context.Context) { - if l.first != nil { - l.first(ctx) - } - for { - select { - case <-ctx.Done(): - l.mu.Lock() - if l.retry != nil { - l.retry.Stop() - } - l.mu.Unlock() - return - case <-l.signal: - } - l.after(l.look(ctx)) - if l.gap > 0 { - select { - case <-ctx.Done(): - case <-time.After(l.gap): - } - } - } -} - -// after takes the outcome of a look: a failed one is looked at again later, a good one forgets the failures. -func (l *looker) after(failed bool) { - l.mu.Lock() - defer l.mu.Unlock() - if !failed { - l.backoff = 0 - if l.retry != nil { - l.retry.Stop() - l.retry = nil - } - return - } - if l.retry != nil { - return // one is armed already - } - switch { - case l.backoff == 0: - l.backoff = l.base - case l.backoff*2 > l.most: - l.backoff = l.most - default: - l.backoff *= 2 - } - // retries: an advance step that failed (store, bus or send). One at a time, five seconds doubling to a minute, forgotten by - // the first look that fails nothing; the step is otherwise started by an event, and a failure is none. - l.retry = time.AfterFunc(l.backoff, func() { - l.mu.Lock() - l.retry = nil - l.mu.Unlock() - l.ask() - }) -} - -// theLooker is the serving controller's; nil in a command, whose looks are its own. -var ( - lookerMu sync.RWMutex - theLooker *looker -) - -// askToLook has the serving controller look at the open plans soon: a plan was saved, a machine reported on a send, -// a silent key expired. Nothing in a process that is not serving. -func askToLook() { - lookerMu.RLock() - l := theLooker - lookerMu.RUnlock() - if l != nil { - l.ask() - } -} - -// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a -// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker. -// It answers a channel that is closed once the look has returned after the context ended, for a test that restores what -// the look reads (issue 1338). -func startPlanLooks(ctx context.Context, open *stores) <-chan struct{} { - l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) }) - l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) } - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - go func() { - <-ctx.Done() - lookerMu.Lock() - if theLooker == l { - theLooker = nil - } - lookerMu.Unlock() - }() - done := make(chan struct{}) - go func() { - defer close(done) - l.run(ctx) - }() - l.ask() - return done -} - -// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's -// ApplySilentAfter, a variable so a test of the wait needs no quarter of an hour. -var applySilentAfter = inventory.ApplySilentAfter - -// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974): -// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own -// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left -// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one. It answers -// a function that waits until the look at the plans has returned, which only a test that ends the context needs (issue -// 1338): the serving controller ignores it. -func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) (wait func()) { - if err := startRolloutSilent(ctx, js); err != nil { - fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err) - } - if err := startProvAppliedLooks(ctx, js); err != nil { - fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err) - } - if err := startPlanMovedLooks(ctx, js, open); err != nil { - fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err) - } - done := startPlanLooks(ctx, open) - return func() { <-done } -} - -// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a -// send kept or a command's move is followed by the look that asks the next tier, in place of the timer that did), and, -// for a walk that ended, its merges' check builds forgotten (novox/hq ADR 0350 §5). -func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inventory.Inventory, inventory.Plan) { - return func(_ *inventory.Inventory, p inventory.Plan) { - sayPlanMoved(ctx, bus, open.inventory, p) - planKept(p) - } -} - -// planLookPasses is the most times one look goes over the plans again because the first pass changed which are open. -const planLookPasses = 6 - -// lookAtPlans moves every open plan as far as the facts allow, as the ticker did each pass: what waits for a gate is -// released, the plans advanced, the wait for each machine's report kept as a key on the bus, and the phases of the -// walks that ended recorded. It says whether a step failed. Waits for the plans rather than giving up: an ask is -// answered once. -func lookAtPlans(ctx context.Context, open *stores) (failed bool) { - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - fmt.Printf("plans: cannot hold them: %v\n", err) - return true - } - func() { - defer release() - // What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353). - if _, err := releaseBacklog(ctx, open, ""); err != nil { - fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) - failed = true - } - // Again while the plans that are open changed: a plan that ended lets a deferred one before it start, and a - // batch cut makes one. - for pass := 0; pass < planLookPasses; pass++ { - before := openPlansNow(ctx, open.inventory) - if advanceHeld(ctx, open) { - failed = true - } - if openPlansNow(ctx, open.inventory) == before { - break - } - } - if err := syncSilentKeys(ctx, open); err != nil { - fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err) - failed = true // so the look is retried - } - }() - keepWalkPhases(ctx, open) - return failed -} - -// advancePlans is one pass over the open plans taking the plans without waiting for them, for a caller that wants a -// single step rather than the look (the tests of the steps). -func advancePlans(ctx context.Context, open *stores) { - release, err := open.inventory.HoldPlans(ctx, false) - if err != nil { - if !errors.Is(err, inventory.ErrPlansBusy) { - fmt.Printf("plans: cannot hold them: %v\n", err) - } - return - } - defer release() - if _, err := releaseBacklog(ctx, open, ""); err != nil { - fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) - } - advanceHeld(ctx, open) -} - -// openPlansNow is which plans are open and in what state, for telling a pass that changed them from one that did not. -func openPlansNow(ctx context.Context, inv *inventory.Inventory) string { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return "unreadable " + time.Now().String() - } - ids := make([]string, 0, len(plans)) - for _, p := range plans { - ids = append(ids, p.ID+":"+string(p.State)) - } - sort.Strings(ids) - return strings.Join(ids, ",") -} - -// --- the wait for a machine's report, as keys on the bus --- - -var ( - silentMu sync.RWMutex - silentKeyed silentClock // nil in a process that is not the serving controller -) - -// setSilentClock makes clock the rollout's; nil takes it away. -func setSilentClock(c silentClock) { - silentMu.Lock() - defer silentMu.Unlock() - silentKeyed = c -} - -func silentClockOf() silentClock { - silentMu.RLock() - defer silentMu.RUnlock() - return silentKeyed -} - -// silentWaitsOf is every machine an open plan waits on for a report, with when it was sent: the machines a walk sent -// that have not reported on the send and are not silent yet (walkReports), the machine a gate of a first machine -// judges, and the machines a release's gates wait on. Pure. -func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[silentWait]time.Time { - waits := map[silentWait]time.Time{} - put := func(w silentWait, at time.Time) { - if at.IsZero() { - return - } - if was, ok := waits[w]; !ok || at.Before(was) { - waits[w] = at - } - } - if !p.Open() { - return waits - } - if r := p.Release; r != nil { - gates := []*inventory.PlanGate{r.Gate} - for _, g := range r.Gates { - gates = append(gates, g) - } - for _, g := range gates { - if g == nil || g.Verdict != "" || g.Silent || g.Pending || g.Since == nil { - continue - } - for _, node := range g.Machines { - put(silentWait{Plan: p.ID, Tier: 0, Node: node}, *g.Since) - } - } - return waits - } - if p.Waiting() { - return waits - } - tierOf := func(module string) int { - for i, tier := range p.Tiers { - if slices.Contains(tier, module) { - return i - } - } - return 0 - } - waiting, _, _ := walkReports(p, reports, now, prov) - for module, s := range p.Modules { - if s == nil { - continue - } - for _, node := range waiting { - sent, ok := s.Rest[node] - if !ok { - continue - } - at := sent.At - if at.IsZero() && s.SentAt != nil { - at = *s.SentAt - } - put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, at) - } - if g := s.Gate; g != nil && g.Verdict == "" && !g.Silent && g.Since != nil && s.FirstAt != nil { - for _, node := range g.Machines { - put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, *g.Since) - } - } - } - return waits -} - -// silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the -// look it wakes finds the machine silent and not a fraction early. -func silentTTL(sent, now time.Time) time.Duration { - left := sent.Add(applySilentAfter).Sub(now) + time.Second - if left < time.Second { - return time.Second - } - return left -} - -// silentDiff is what the keys on the bus need to be as the waits are: the waits to put a key for (none yet, or the -// key is of an earlier send) and the keys to delete (the wait is over). Pure. -func silentDiff(want, held map[silentWait]time.Time) (arm, drop []silentWait) { - for w, sent := range want { - if was, ok := held[w]; !ok || !was.Equal(sent) { - arm = append(arm, w) - } - } - for w := range held { - if _, ok := want[w]; !ok { - drop = append(drop, w) - } - } - return arm, drop -} - -// syncSilentKeys keeps the keys on the bus as the open plans' waits are: a wait without a key gets one that lives out -// its time, a send made again gets its key again, and a key whose wait is over — the machine reported on the send, the -// gate has its verdict, the plan ended — is deleted. A report deletes the key by way of the look it asks for (the -// look alone knows the report was on that send). Only the lease holder looks, so only it writes. -func syncSilentKeys(ctx context.Context, open *stores) error { - c := silentClockOf() - inv := open.inventory - plans, err := inv.OpenPlans(ctx) - if err != nil { - return err - } - reports, err := inv.LastReports(ctx) - if err != nil { - return err - } - now := time.Now().UTC() - want := map[silentWait]time.Time{} - for _, p := range plans { - for w, at := range silentWaitsOf(p, reports, now, provLookup(ctx)) { - want[w] = at - } - } - return keepSilentKeys(ctx, c, want, now) -} - -// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys -// cannot be read, one cannot be put — the refusal is raised as a condition naming the plan and the machine, never -// stood in for by a timer, and the error is given back so the look is retried and writes the key again (novox/hq ADR -// 0433 §4). The set of refusals is what this look found: a key written at the next look is no longer in it. -func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error { - refused := func(err error) map[silentWait]error { - out := map[silentWait]error{} - for w := range want { - out[w] = err - } - return out - } - if c == nil { - noteRefusedSendKeys(refused(errors.New("the bus's clock for the waits is not there"))) - return nil - } - c = boundedClock{c} - held, err := c.Held(ctx) - if err != nil { - noteRefusedSendKeys(refused(err)) - return err - } - arm, drop := silentDiff(want, held) - var first error - notTaken := map[silentWait]error{} - for _, w := range arm { - if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil { - notTaken[w] = err - if first == nil { - first = err - } - } - } - noteRefusedSendKeys(notTaken) - for _, w := range drop { - if err := c.Reported(ctx, w); err != nil && first == nil { - first = err - } - } - return first -} - -// silentCallBound is how long one call to the silent clock may take while the look holds the plans (novox/hq issue 1121). -// The vendored key listing reads its watcher without the context, so a consumer lost when the broker reloaded its user -// list left the look waiting for good, with the plans and every node held. -var silentCallBound = 20 * time.Second - -// boundedClock is a clock whose every call is answered within silentCallBound, or fails: a call that never returns is -// left behind on its own goroutine. -type boundedClock struct{ silentClock } - -func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) { - ctx, cancel := context.WithTimeout(ctx, silentCallBound) - defer cancel() - type answer struct { - v T - err error - } - done := make(chan answer, 1) - go func() { v, err := call(ctx); done <- answer{v, err} }() - select { - case a := <-done: - return a.v, a.err - case <-ctx.Done(): - var zero T - return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err()) - } -} - -func (b boundedClock) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error { - _, err := boundedCall(ctx, "arming a wait", func(ctx context.Context) (struct{}, error) { - return struct{}{}, b.silentClock.Arm(ctx, w, ttl, sent) - }) - return err -} - -func (b boundedClock) Held(ctx context.Context) (map[silentWait]time.Time, error) { - return boundedCall(ctx, "listing the waits", b.silentClock.Held) -} - -func (b boundedClock) Reported(ctx context.Context, w silentWait) error { - _, err := boundedCall(ctx, "dropping a wait", func(ctx context.Context) (struct{}, error) { - return struct{}{}, b.silentClock.Reported(ctx, w) - }) - return err -} - -// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the -// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires. -func startRolloutSilent(ctx context.Context, js *broker.JetStream) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - clock, err := newBusSilent(ctx, api, func(silentWait) { askToLook() }) - if err != nil { - return err - } - setSilentClock(clock) - go func() { - <-ctx.Done() - silentMu.Lock() - if silentKeyed == silentClock(clock) { - silentKeyed = nil - } - silentMu.Unlock() - }() - return nil -} - -// startProvAppliedLooks has the plans looked at when a machine's provisioner client says a word (`prov.applied..`): -// a send that handed modules to the provisioner is applied only once the client said so too, and without this the walk -// would wait for its silent key (ADR 0362, ADR 0406 §5, issue 974). New words only: the first look reads the rest. -func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - stream, err := api.Stream(ctx, broker.ProvAppliedStream) - if err != nil { - return fmt.Errorf("the provisioner clients' words: %w", err) - } - consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverNewPolicy}) - if err != nil { - return fmt.Errorf("watching the provisioner clients' words: %w", err) - } - consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() }) - if err != nil { - return fmt.Errorf("watching the provisioner clients' words: %w", err) - } - go func() { - <-ctx.Done() - consuming.Stop() - }() - return nil -} - -// startPlanMovedLooks has the plans looked at when any process says one moved: the serving controller follows its own -// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn, -// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the -// first look reads the rest. A look its own save asks is the same look: asks are one. -func startPlanMovedLooks(ctx context.Context, js *broker.JetStream, open *stores) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - stream, err := api.Stream(ctx, broker.EventsStream) - if err != nil { - return fmt.Errorf("the events stream: %w", err) - } - consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{ - FilterSubjects: []string{link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved)}, - DeliverPolicy: jetstream.DeliverNewPolicy}) - if err != nil { - return fmt.Errorf("following the plans moved: %w", err) - } - var bus link.Bus - if core, err := js.Conn().JetStream(); err == nil { - bus = link.OverNATS{Conn: js.Conn(), JS: core} - } - consuming, err := consumer.Consume(func(m jetstream.Msg) { - askToLook() - if bus != nil && open != nil { - // A walk's start or end, said by this controller or a verb: said again with the machines whose - // declaration it changes (novox/hq ADR 0505 §2, walk_declared.go). - go sayDeclared(context.WithoutCancel(ctx), bus, open, m.Data()) - } - }) - if err != nil { - return fmt.Errorf("following the plans moved: %w", err) - } - go func() { - <-ctx.Done() - consuming.Stop() - }() - return nil -} - -// --- the end of a walk whose end is not known --- - -var walkEnds = struct { - mu sync.Mutex - timers map[string]*time.Timer -}{timers: map[string]*time.Timer{}} - -// armWalkEnd has the walk recorded once its end can be known: a walk that ended with a machine silent is recorded -// without its total ApplySilentAfter and a minute after its last save (recordWalkPhases), which no event says. Armed at -// each save of the done walk, so it counts from the last. -func armWalkEnd(id string, savedAt time.Time) { - d := time.Until(savedAt.Add(inventory.ApplySilentAfter + time.Minute + time.Second)) - if d <= 0 { - return - } - walkEnds.mu.Lock() - defer walkEnds.mu.Unlock() - if t := walkEnds.timers[id]; t != nil { - t.Stop() - } - // retries: recording the end of a walk whose end is not known. One per walk, once, from its last save; the first - // look after the lease was taken arms it again for the walks still unrecorded. - walkEnds.timers[id] = time.AfterFunc(d, func() { - walkEnds.mu.Lock() - delete(walkEnds.timers, id) - walkEnds.mu.Unlock() - askToLook() - }) -} - -// rearmWalkEnds is armWalkEnd for the walks that ended lately, for a controller that has just taken the lease. -func rearmWalkEnds(ctx context.Context, open *stores) { - recent, err := open.inventory.RecentPlans(ctx, 30) - if err != nil { - fmt.Printf("plans: the walks ended lately could not be read to record their phases: %v\n", err) - return - } - for _, p := range recent { - if p.State == inventory.PlanDone && p.Release == nil && time.Since(p.Updated) <= 2*time.Hour { - armWalkEnd(p.ID, p.Updated) - } - } -} - -// planKept is what the serving controller does when a plan was saved: a look is asked for, so a send, a built tier or -// a command's move is followed by the plan's next step; and a walk that ended is armed to be recorded. -func planKept(p inventory.Plan) { - if p.State == inventory.PlanDone && p.Release == nil { - armWalkEnd(p.ID, time.Now()) - } - askToLook() -} - -// --- `sent`: a tier sent to a machine, said --- - -// sendMade is one machine a plan's tier was sent to. -type sendMade struct { - Tier int - Node string - Modules []string - At time.Time -} - -// sendsMade collects the sends one step of a plan made, to be said once the plan is saved. -type sendsMade struct { - mu sync.Mutex - list []sendMade -} - -type sendsMadeKey struct{} - -func withSendsMade(ctx context.Context, s *sendsMade) context.Context { - return context.WithValue(ctx, sendsMadeKey{}, s) -} - -// noteSends records that modules of the plan's tier were sent to the nodes. -func noteSends(ctx context.Context, tier int, nodes, modules []string, at time.Time) { - s, _ := ctx.Value(sendsMadeKey{}).(*sendsMade) - if s == nil { - return - } - s.mu.Lock() - defer s.mu.Unlock() - for _, n := range nodes { - s.list = append(s.list, sendMade{Tier: tier, Node: n, Modules: slices.Clone(modules), At: at}) - } -} - -// planSentBody is the body of the seat event `sent` (novox/hq ADR 0406 §5, issue 974): the plan, the tier, the machine -// and the modules it was sent, and when. -type planSentBody struct { - Event string `json:"event"` - Plan string `json:"plan"` - Tier int `json:"tier"` - Node string `json:"node"` - Modules []string `json:"modules"` - At time.Time `json:"at"` -} - -// sentBodies are the events a step's sends are said as. -func sentBodies(plan string, sends *sendsMade) []planSentBody { - sends.mu.Lock() - defer sends.mu.Unlock() - out := make([]planSentBody, 0, len(sends.list)) - for _, s := range sends.list { - out = append(out, planSentBody{Event: link.KeySent, Plan: plan, Tier: s.Tier, Node: s.Node, - Modules: s.Modules, At: s.At}) - } - return out -} - -// saySends says each send a step made as `sent`, once the plan that records it is saved. What an outsider hears; the -// controller's own next step does not wait for it. -func saySends(ctx context.Context, p inventory.Plan, sends *sendsMade) { - bus := checkEvents - if bus == nil { - return - } - for _, body := range sentBodies(p.ID, sends) { - raw, err := json.Marshal(body) - if err != nil { - continue - } - go func() { - stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer stop() - if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySent, raw); err != nil { - fmt.Printf("%s: tier %d was sent to %s, and could not be said as sent: %v\n", body.Plan, body.Tier, body.Node, err) - } - }() - } -} diff --git a/cmd/mesh-controller/plan_looks_integration_test.go b/cmd/mesh-controller/plan_looks_integration_test.go deleted file mode 100644 index cf2f75fb..00000000 --- a/cmd/mesh-controller/plan_looks_integration_test.go +++ /dev/null @@ -1,188 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) { - var failures, looks atomic.Int32 - block := make(chan struct{}) - l := lookerCounting(&failures, &looks, block) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "the first look starts", func() bool { return looks.Load() == 1 }) - for i := 0; i < 20; i++ { - l.ask() - } - block <- struct{}{} // the first look ends - eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 }) - block <- struct{}{} - time.Sleep(60 * time.Millisecond) - if n := looks.Load(); n != 2 { - t.Errorf("%d looks for 21 asks; want 2", n) - } -} - -func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) { - var failures, looks atomic.Int32 - failures.Store(3) // three looks fail, then one is good - l := lookerCounting(&failures, &looks, nil) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "four looks", func() bool { return looks.Load() == 4 }) - time.Sleep(200 * time.Millisecond) - if n := looks.Load(); n != 4 { - t.Errorf("%d looks; the good one should have ended the retries at 4", n) - } - l.mu.Lock() - armed, backoff := l.retry != nil, l.backoff - l.mu.Unlock() - if armed || backoff != 0 { - t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff) - } -} - -func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) { - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - - planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone}) - t.Cleanup(func() { - walkEnds.mu.Lock() - defer walkEnds.mu.Unlock() - if t := walkEnds.timers["walk-1"]; t != nil { - t.Stop() - } - delete(walkEnds.timers, "walk-1") - }) - eventually(t, "a look", func() bool { return looks.Load() == 1 }) - walkEnds.mu.Lock() - _, armed := walkEnds.timers["walk-1"] - walkEnds.mu.Unlock() - if !armed { - t.Error("a walk saved done is not armed to be recorded") - } - askToLook() // no panic, no looker, once it is taken away -} - -func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startRolloutSilent(ctx, js); err != nil { - t.Fatal(err) - } - c := silentClockOf() - w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"} - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := c.Reported(ctx, w); err != nil { - t.Fatal(err) - } - time.Sleep(1800 * time.Millisecond) - if n := looks.Load(); n != 0 { - t.Fatalf("a key a report deleted asked for %d looks", n) - } - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) -} - -func TestAProvisionerClientsWordAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startProvAppliedLooks(ctx, js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 }) -} - -func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startPlanMovedLooks(ctx, js, nil); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 }) -} diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go deleted file mode 100644 index 0d62b8fb..00000000 --- a/cmd/mesh-controller/plan_looks_test.go +++ /dev/null @@ -1,265 +0,0 @@ -package main - -import ( - "context" - "errors" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) { - sent := time.Now().Add(-sentAgo).UTC() - p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0, - Modules: map[string]*inventory.PlanModule{ - "a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, - "b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, - }} - r := inventory.Reported{Node: "ace", Sent: &sent} - if reported { - at := sent.Add(time.Minute) - r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied - } - return p, []inventory.Reported{r} -} - -// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR -// 0406 §5); one that reported on the send, or that is silent already, is none. -func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) { - now := time.Now().UTC() - p, reports := rolloutOf(5*time.Minute, false) - waits := silentWaitsOf(p, reports, now, nil) - if len(waits) != 1 { - t.Fatalf("waits = %v; want the one machine", waits) - } - for w, at := range waits { - if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute { - t.Errorf("wait %+v sent at %v is wrong", w, at) - } - } - p, reports = rolloutOf(5*time.Minute, true) - if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { - t.Errorf("a machine that reported on its send is waited on: %v", waits) - } - p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false) - if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { - t.Errorf("a machine silent past its time is waited on again: %v", waits) - } - p, reports = rolloutOf(time.Minute, false) - p.State = inventory.PlanDone - if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { - t.Errorf("a plan that ended is waiting: %v", waits) - } -} - -// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait. -func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) { - since := time.Now().Add(-time.Minute).UTC() - g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since} - p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}} - waits := silentWaitsOf(p, nil, time.Now(), nil) - if len(waits) != 2 { - t.Fatalf("waits = %v; want g14 and novox", waits) - } - g.Verdict = inventory.GatePassed - if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 { - t.Errorf("a gate with its verdict still waits: %v", waits) - } -} - -// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted. -func TestTheKeysFollowTheWaits(t *testing.T) { - t1 := time.Now().UTC() - t2 := t1.Add(time.Minute) - a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"} - arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1}) - if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) { - t.Errorf("armed %v; want b (sent again) and c (no key)", arm) - } - if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) { - t.Errorf("dropped %v; want the key with no wait", drop) - } - if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second { - t.Errorf("a wait long over has a key of %v; want the least, one second", ttl) - } -} - -func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker { - l := newLooker(func(ctx context.Context) bool { - looks.Add(1) - if block != nil { - <-block - } - return failures.Add(-1) >= 0 - }) - l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond - return l -} - -func eventually(t *testing.T, what string, ok func() bool) { - t.Helper() - for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { - if ok() { - return - } - } - t.Fatalf("never: %s", what) -} - -// Asks made while a look runs are not lost and are one look after it, not one each. - -// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails -// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry). - -func TestTheRetryWaitDoublesToItsMost(t *testing.T) { - l := newLooker(func(context.Context) bool { return true }) - l.base, l.most = 5*time.Second, time.Minute - var seen []time.Duration - for i := 0; i < 6; i++ { - l.after(true) - seen = append(seen, l.backoff) - l.retry.Stop() - l.retry = nil - } - want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute} - for i := range want { - if seen[i] != want[i] { - t.Fatalf("waits %v; want %v", seen, want) - } - } -} - -// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known. - -// The tier-sends a step made are said once each, per machine, with the modules it was sent. -func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { - sends := &sendsMade{} - ctx := withSendsMade(context.Background(), sends) - at := time.Now().UTC() - noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at) - noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at) - noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept - got := sentBodies("plan-9", sends) - if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" || - got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" { - t.Errorf("sent bodies = %+v", got) - } -} - -// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5). - -// A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only -// once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said. - -type brokenClock struct{ silentClock } - -func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) { - return nil, errors.New("the bus is down") -} - -// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by -// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error -// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4). -func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1 - w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"} - sent := time.Now().UTC() - for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{}, - "no clock": nil} { - t.Run(name, func(t *testing.T) { - noteRefusedSendKeys(nil) - t.Cleanup(func() { noteRefusedSendKeys(nil) }) - err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent) - if clock != nil && err == nil { - t.Fatal("no error given back") - } - got := observeRefusedSendKeys() - if len(got) != 1 { - t.Fatalf("observations %+v", got) - } - o := got[0] - if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused || - !strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") { - t.Fatalf("the condition does not name the plan and the machine: %+v", o) - } - }) - } -} - -// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is. -func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2 - w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"} - sent := time.Now().UTC() - noteRefusedSendKeys(nil) - t.Cleanup(func() { noteRefusedSendKeys(nil) }) - want := map[silentWait]time.Time{w: sent} - if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 { - t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys()) - } - clock := &keptClock{} - if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil { - t.Fatal(err) - } - if !clock.armed[w].Equal(sent) { - t.Fatalf("the next look did not write the key again: %v", clock.armed) - } - if got := observeRefusedSendKeys(); len(got) != 0 { - t.Fatalf("the condition stays after the key was written: %+v", got) - } -} - -type armRefusingClock struct{ silentClock } - -func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil } -func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error { - return errors.New("the bus refuses the key") -} - -// keptClock is a clock that takes every key. -type keptClock struct { - silentClock - armed map[silentWait]time.Time -} - -func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil } -func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error { - if c.armed == nil { - c.armed = map[silentWait]time.Time{} - } - c.armed[w] = sent - return nil -} - -// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is -// looked at: the controller follows its own plan-moved (issue 974, round 1). - -// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that -// never reaches its end. -type hungClock struct{ silentClock } - -func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) { - select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it) -} - -// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans -// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition. -func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) { - was := silentCallBound - silentCallBound = 50 * time.Millisecond - t.Cleanup(func() { silentCallBound = was }) - w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"} - sent := time.Now().UTC() - done := make(chan error, 1) - go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }() - select { - case err := <-done: - if err == nil { - t.Fatal("a clock that never answered was reported as fine") - } - case <-time.After(5 * time.Second): - t.Fatal("the look is still waiting for a clock that never answers") - } -} diff --git a/cmd/mesh-controller/plan_moved_every_writer_integration_test.go b/cmd/mesh-controller/plan_moved_every_writer_integration_test.go deleted file mode 100644 index 2d376641..00000000 --- a/cmd/mesh-controller/plan_moved_every_writer_integration_test.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" -) - -func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - heard := sayingEverySave(t) - hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second)) - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batch(es) opened", len(bs)) - } - heard.noUnsaid(t, open) - cutAt(t, open, t0.Add(91*time.Second)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs)) - } - heard.noUnsaid(t, open) - counted := 0 - for _, r := range heard.revs { - counted += len(r) - } - cutAt(t, open, t0.Add(92*time.Second)) - again := 0 - for _, r := range heard.revs { - again += len(r) - } - if again != counted { - t.Errorf("a look that changed nothing said %d more move(s)", again-counted) - } -} diff --git a/cmd/mesh-controller/plan_moved_every_writer_test.go b/cmd/mesh-controller/plan_moved_every_writer_test.go deleted file mode 100644 index 1b68d3da..00000000 --- a/cmd/mesh-controller/plan_moved_every_writer_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "regexp" - "strings" - "sync" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan -// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the -// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts -// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed. -func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { - write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`) - var found []string - for _, root := range []string{"../../cmd", "../../internal"} { - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { - return nil - } - raw, err := os.ReadFile(path) - if err != nil { - return err - } - text := string(raw) - // The regular expression is the costly part of reading every source: only a file that names the table can match. - if strings.Contains(strings.ToLower(text), "release_plan") && write.MatchString(text) { - found = append(found, path+" writes the plan table") - } - // A plan record is written by the bucket's raw create and update, which only the plans' own file calls. - if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" && - strings.Contains(text, "chain.RolloutsOf") { - found = append(found, path+" reaches the rollouts bucket") - } - return nil - }) - if err != nil { - t.Fatal(err) - } - } - if len(found) > 0 { - t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n")) - } - raw, err := os.ReadFile("../../internal/inventory/plans.go") - if err != nil { - t.Fatal(err) - } - text := string(raw) - save := strings.Index(text, "func (i *Inventory) SavePlansMoving") - next := strings.Index(text[save:], "\nfunc ") - if next < 0 { - next = len(text) - save - } - if !strings.Contains(text[save:save+next], "chainSavePlans(") { - t.Error("SavePlansMoving does not write through chainSavePlans") - } - chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go") - if err != nil { - t.Fatal(err) - } - chainText := string(chainRaw) - at := strings.Index(chainText, "func (i *Inventory) chainSavePlans") - end := strings.Index(chainText[at:], "\nfunc ") - if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") { - t.Error("chainSavePlans does not say each plan it wrote") - } - if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 { - t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n) - } -} - -// movesSaid records what the hook hears: for each plan, the revisions it was said at. -type movesSaid struct { - mu sync.Mutex - revs map[string][]int64 -} - -func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) { - s.mu.Lock() - defer s.mu.Unlock() - s.revs[p.ID] = append(s.revs[p.ID], p.Revision) -} - -// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved. -func sayingEverySave(t *testing.T) *movesSaid { - t.Helper() - s := &movesSaid{revs: map[string][]int64{}} - was := inventory.PlanSaved - inventory.PlanSaved = s.hear - t.Cleanup(func() { inventory.PlanSaved = was }) - return s -} - -// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said. -func (s *movesSaid) noUnsaid(t *testing.T, open *stores) { - t.Helper() - recent, err := open.inventory.RecentPlans(t.Context(), 50) - if err != nil { - t.Fatal(err) - } - s.mu.Lock() - defer s.mu.Unlock() - if len(recent) == 0 { - t.Fatal("no plan was kept") - } - for _, p := range recent { - revs := s.revs[p.ID] - if len(revs) == 0 || revs[len(revs)-1] != p.Revision { - t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs) - } - } -} - -// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that -// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2). diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go deleted file mode 100644 index f7bf2a41..00000000 --- a/cmd/mesh-controller/plan_retry.go +++ /dev/null @@ -1,651 +0,0 @@ -package main - -import ( - "context" - "fmt" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A plan follows what is done to the build queue (novox/hq ADR 0219). -// -// Two things a person does to builds by hand would otherwise leave a plan saying something untrue: -// -// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder -// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE -// sends a reader looking for a fault that is a decision. It says what it waits on instead. -// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a -// failed plan's failed modules and the plan goes on from that tier as if they had built the -// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running -// beside it — beside it, the plan would either ask it again or stay failed on an outcome the -// rebuild has already replaced. - -// failedIn is the modules of a plan's current tier that failed to build, sorted. -func failedIn(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.State == "failed" { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan -// that holds what this one held now (issue 254, ADR 0218). -func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { - for _, q := range plans { - if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) || - (p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) { - continue - } - return q, true - } - return inventory.Plan{}, false -} - -// newerDonePlan is a done plan of the same repository and branch made after this one: the newer commit it delivered -// runs now (novox/hq ADR 0451). -func newerDonePlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { - var found inventory.Plan - var ok bool - for _, q := range plans { - if q.ID == p.ID || q.State != inventory.PlanDone || !strings.EqualFold(q.Repository, p.Repository) || - (p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) { - continue - } - if !ok || q.Created.After(found.Created) { - found, ok = q, true - } - } - return found, ok -} - -// passedBy is the refusal of a retry of a plan that a newer done plan of its repository and branch has passed. -type passedBy struct{ plan, by, repository, commit string } - -func (e passedBy) Error() string { - return fmt.Sprintf("%s was passed by %s, a newer plan of %s (at %s) that is done: retrying it would build and "+ - "send the older commit over the newer one that runs", e.plan, e.by, e.repository, short(e.commit)) -} - -// retryRefusal is why a plan cannot be retried, or nothing. -func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error { - switch { - case p.State == inventory.PlanDone: - return fmt.Errorf("%s is done; there is nothing to retry", p.ID) - case p.State == inventory.PlanSuperseded: - return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note) - case p.Open(): - return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild ` asks one module again", p.ID, p.State) - } - - // **A newer delivery of the branch has passed it** (novox/hq ADR 0451): retrying would build and send the older - // commit over the newer one that runs. - if q, found := newerDonePlan(p, plans); found { - return passedBy{plan: p.ID, by: q.ID, repository: q.Repository, commit: q.Commit} - } - if len(failedIn(p)) > 0 { - if q, found := newerOpenPlan(p, plans); found { - return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+ - "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) - } - return oneWalkAtATime(p, plans, onPath) - } - stopped := stoppedRollouts(p) - // **A walk stopped by hand while its tier waited is resumed** (issue 1886): nothing in it failed, so nothing is - // asked again; it goes on from where it stopped, unless a newer walk holds what it has yet to send. - if len(stopped) == 0 && stoppedByHandWaiting(p) { - if q, found := newerOpenPlan(p, plans); found { - return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would build "+ - "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) - } - for _, m := range holdingABuild(p) { - if q, found := newerPlanFor(m, p, plans); found { - return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): resuming %s would send the older build of it "+ - "over the newer one", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID) - } - } - return oneWalkAtATime(p, plans, onPath) - } - // **A walk lost to silence is resumed** (novox/hq ADR 0531, issue 1886): its tier was sent whole and only the gate after it failed it, every - // machine it waited on silent; once they reported, it goes on from that gate. - if len(stopped) == 0 && lostToSilence(p) { - if q, found := newerOpenPlan(p, plans); found { - return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would send "+ - "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) - } - return oneWalkAtATime(p, plans, onPath) - } - if len(stopped) == 0 { - return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s", - p.Tier, p.ID, p.Note) - } - // **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first - // machine, and retrying would judge the build the mesh put back, or send the failed one by hand. - for _, m := range stopped { - if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed && !noVerdictOnItsBuild(g) { - return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+ - "sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again", - m, strings.Join(g.Machines, ", "), g.Why, m) - } - } - // **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or - // sent — a newer build, and sending this one again would put the older build back on its machines. - for _, m := range stopped { - if q, found := newerPlanFor(m, p, plans); found { - return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+ - "the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID) - } - } - return oneWalkAtATime(p, plans, onPath) -} - -// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq -// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same -// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it -// plans, which already hold their dependents, and the repositories it carries. -func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error { - mine := footprintOfWalk(p, nil) - for _, q := range plans { - if q.ID == p.ID || !q.Open() || q.Release != nil { - continue - } - if ownWalk(p, onPath) || ownWalk(q, onPath) { - return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+ - "retry %s once it ended", q.ID, q.Named(), p.ID) - } - if ok, why := related(mine, footprintOfWalk(q, nil), nil); ok { - return fmt.Errorf("%s is open (%s) and %s: one walk at a time beside a related one — retry %s once it ended", - q.ID, q.Named(), why, p.ID) - } - } - return nil -} - -// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine -// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept. -func stoppedRollouts(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil && - len(s.First) > 0 && s.Why != "" { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// newerPlanFor is a plan made after this one that holds the module, superseded ones aside. -func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { - for _, q := range plans { - if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) { - continue - } - for _, tier := range q.Tiers { - for _, m := range tier { - if m == module { - return q, true - } - } - } - } - return inventory.Plan{}, false -} - -// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A -// variable so a test of a retried rollout needs no machine. -var sendRollout = sendToEach - -// resumed sets a failed plan building again once nothing in its tier is failed. -func resumed(p *inventory.Plan, why string) { - if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 { - p.State = inventory.PlanBuilding - p.Note = why - goesOnAgain(p) - } -} - -// goesOnAgain clears a stop from a walk that walks again: a stop said why the walk ended, and a walk resumed has -// not ended. Kept, it would end a retried walk that fails again as stopped, not failed (novox/hq ADR 0308 -// decision 1), and keep the search for the merge that brought the failure from running (searchAfterFailures). -func goesOnAgain(p *inventory.Plan) { - if p.Delivery != nil { - p.Delivery.Stopped, p.Delivery.StoppedWhy = "", "" - } -} - -// planReader is what a retry reads the mesh's plans through. -type planReader interface { - OpenPlans(ctx context.Context) ([]inventory.Plan, error) - RecentPlans(ctx context.Context, limit int) ([]inventory.Plan, error) - PlansSince(ctx context.Context, since time.Time) ([]inventory.Plan, error) -} - -// plansForRetry are the plans a retry of p judges itself against. -func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inventory.Plan, error) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - recent, err := inv.RecentPlans(ctx, 50) - if err != nil { - return nil, err - } - // **Every plan made after this one** (novox/hq ADR 0451 §1): a plan that passed it is seen however many plans - // were made since, not only when it is among the newest. - later, err := inv.PlansSince(ctx, p.Created) - if err != nil { - return nil, err - } - return append(append(plans, recent...), later...), nil -} - -// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at -// that tier: what follows is the plan going on as if they had built the first time. -// A walk stopped by hand while its tier waited, with nothing failed, is resumed instead: nothing asked again -// (resumeStopped, issue 1886). -func retryPlan(ctx context.Context, open *stores, id string) (string, error) { - inv := open.inventory - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return "", err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return "", err - } - plans, err := plansForRetry(ctx, inv, p) - if err != nil { - return "", err - } - // **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for - // the merge that brought the failure answers them now, and a retried walk would name them twice. - // **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped, - // and the others are another walk's now. - if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) { - return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+ - "newer merge, or `rebuild `, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut), - orWords(p.Delivery.WalkedAgainBy, "nothing")) - } - if p.Delivery != nil && len(p.Delivery.Merges) > 0 { - kept, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return "", err - } - if len(kept) < len(p.Delivery.Merges) { - return "", fmt.Errorf("%s's earlier merges are walked alone, to find which one brought its failure: "+ - "those walks answer them; a newer merge, or `rebuild `, builds again", p.ID) - } - } - // Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier - // that failed after the plan did is as failed as the one that failed it. What toRetry says is the - // failed set every step below works from. - var failed []string - if p.Tier < len(p.Tiers) { - recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier]) - if err != nil { - return "", err - } - failed = toRetry(&p, recorded, byID) - } - onPath, err := ownPathNow(ctx, inv) - if err != nil { - return "", err - } - if err := retryRefusal(p, plans, onPath); err != nil { - return "", err - } - if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 { - return retryTierWhole(ctx, open, &p, again) - } - if len(failed) == 0 && len(stoppedRollouts(p)) == 0 && stoppedByHandWaiting(p) { - said := resumeStopped(&p) - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - return said, nil - } - if len(failed) == 0 && lostToSilence(p) { - reports, err := inv.LastReports(ctx) - if err != nil { - return "", err - } - said, err := resumeLost(&p, silentNodes(p, reports, time.Now().UTC(), provLookup(ctx))) - if err != nil { - return "", err - } - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - return said, nil - } - if len(failed) == 0 { - return retryRollouts(ctx, open, &p) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return "", err - } - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - var asked []string - for _, m := range failed { - askModule(ctx, &p, m, byName) - if s := p.Modules[m]; s.State == "asked" && s.Key { - asked = append(asked, m+" from the builder's key") - } else if s.State == "asked" { - asked = append(asked, m+" as "+s.Build) - } - } - waiting := busWaitingOnRetry(p) - note, said := retryWords(p, failed, asked, waiting) - resumed(&p, note) - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - if p.State != inventory.PlanBuilding { - return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note) - } - return said, nil -} - -// busWaitingOnRetry are the modules a retry left waiting for the bus. Read before the plan is resumed, while it still says -// failed, so read as the plan the retry resumes: a failed plan is not open, and busWaitingIn would name none, leaving the -// answer "asked" with nothing after it (issue 1403). -func busWaitingOnRetry(p inventory.Plan) []string { - p.State = inventory.PlanBuilding - return busWaitingIn(p) -} - -// retryWords are the plan's note and the verb's answer for a retry: what was asked again (as the builds it got), and what waits -// for the bus. A retry whose modules all wait for the bus asked none, and says so rather than "asked" with nothing after it -// (issue 1403). -func retryWords(p inventory.Plan, failed, asked, waiting []string) (note, said string) { - again := slicesWithout(failed, waiting) - switch { - case len(again) > 0: - note = fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(again, ", ")) - said = fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does", - p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")) - default: - note = fmt.Sprintf("tier %d retried by hand: nothing could be asked yet", p.Tier) - said = fmt.Sprintf("%s retried at tier %d of %d: nothing asked yet; the plan goes on from there as any plan does", - p.ID, p.Tier, len(p.Tiers)) - } - if len(waiting) > 0 { - note += "; " + busWaitWords(p, waiting) - said += "; " + busWaitWords(p, waiting) - } - return note, said -} - -// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again, -// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan -// would have — the rest sent once those report they applied it, the next tier after (ADR 0218). -func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) { - // Every machine once, for all the modules stopped there (novox/hq issue 281). - stopped := stoppedRollouts(*p) - var machines []string - for _, m := range stopped { - for _, n := range p.Modules[m].First { - if !slices.Contains(machines, n) { - machines = append(machines, n) - } - } - } - sort.Strings(machines) - sent, err := sendRollout(ctx, open, machines) - if err != nil { - return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w", - strings.Join(stopped, ", "), strings.Join(machines, ", "), p.ID, err) - } - now := time.Now().UTC() - var said []string - for _, m := range stopped { - s := p.Modules[m] - var again []string - for _, n := range sent { - if slices.Contains(s.First, n) { - again = append(again, n) - } - } - s.First, s.FirstAt, s.Why = again, &now, "" - said = append(said, m+" to "+strings.Join(again, ", ")) - } - p.State = inventory.PlanRolling - goesOnAgain(p) - p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; ")) - if err := open.inventory.SavePlan(ctx, p); err != nil { - return "", err - } - return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+ - "applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil -} - -// noVerdictOnItsBuild says a failed gate said nothing about the module's build (novox/hq issue 281): its -// send changed nothing of the module there — the machine already ran that build, carried there by an -// earlier send of the same tier, or one identical to it. Such a module was blamed for its machine. -func noVerdictOnItsBuild(g *inventory.PlanGate) bool { - return g.Rollback == gateUnchanged || (g.From != "" && sameCommit(g.From, g.To)) -} - -// unjudgedAtGate is the modules of a plan's current tier stopped at a gate that was no verdict on their -// build, sorted. -func unjudgedAtGate(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.Gate != nil && s.Gate.Verdict == inventory.GateFailed && noVerdictOnItsBuild(s.Gate) { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// retryTierWhole retries a plan stopped at a gate that judged no build of the module it stopped on -// (issue 281): that module is asked again under a new id — its old build may be marked failed, and a new -// verdict is what takes the gate's condition away — and every module of the tier sent first and never -// passed is sent again, the tier whole, one send per machine, judged again. What passed stays passed. -func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again []string) (string, error) { - inv := open.inventory - entries, err := inv.Catalogued(ctx) - if err != nil { - return "", err - } - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - var resent []string - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - if s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(again, m) || - (s.Gate != nil && s.Gate.Verdict == inventory.GatePassed) { - continue - } - sendAgain(s) - resent = append(resent, m) - } - var asked []string - for _, m := range again { - sendAgain(p.Modules[m]) - askModule(ctx, p, m, byName) - if s := p.Modules[m]; s.State == "asked" && s.Key { - asked = append(asked, m+" from the builder's key") - } else if s.State == "asked" { - asked = append(asked, m+" as "+s.Build) - } - } - if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 { - p.State = inventory.PlanBuilding - goesOnAgain(p) - p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier, - strings.Join(again, ", "), orNone(strings.Join(resent, ", "))) - } - if err := inv.SavePlan(ctx, p); err != nil { - return "", err - } - if p.State != inventory.PlanBuilding { - return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note) - } - return fmt.Sprintf("%s retried at tier %d of %d: asked %s; %s sent again with the tier, one send per machine, "+ - "judged again", p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", "), orNone(strings.Join(resent, ", "))), nil -} - -// sendAgain forgets a module's first send, so its plan sends it again. -func sendAgain(s *inventory.PlanModule) { - s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", "" -} - -// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state -// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its -// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build -// records. Read from the plan alone, a retry asked only the build that failed first, and the records -// then failed the plan again on the next: each failed build of a tier took a retry of its own. What -// still runs is left asked, and its outcome is the plan's once the retry sets it building. -func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string { - if p.Tier < len(p.Tiers) { - settleFromRecords(p, p.Tiers[p.Tier], recorded, byID) - } - return failedIn(*p) -} - -// stoppedByHandWaiting says a walk was stopped by hand while its current tier still waited (issue 1886): stopped by a -// person, nothing of the tier failed, and a module of it asked, built and not sent, or not yet asked. A stop that left -// merges out is not one: retryPlan refuses it before this is read (novox/hq ADR 0299). -func stoppedByHandWaiting(p inventory.Plan) bool { - if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped == "" || - p.Tier >= len(p.Tiers) || len(failedIn(p)) > 0 { - return false - } - asked, unsent, unasked := waitingIn(p) - return asked+unsent+unasked > 0 -} - -// waitingIn counts what a plan's current tier waits on: modules asked, built and not sent, not yet asked. -func waitingIn(p inventory.Plan) (asked, unsent, unasked int) { - if p.Tier >= len(p.Tiers) { - return 0, 0, 0 - } - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - switch { - case s == nil || s.State == "": - unasked++ - case s.State == "asked": - asked++ - case s.State == "built" && s.SentAt == nil: - unsent++ - } - } - return asked, unsent, unasked -} - -// holdingABuild is the modules of a stopped walk's current tier that hold a build of its commit, not yet sent: built and -// not sent, or asked under a build id. Only these could put an older build over a newer plan's (review of issue 1886): a -// module not yet asked is asked at its followed branch's head when the walk reaches it, and one asked on the builder's key -// is settled by the look from the key of the commit it reads then. Sorted. -func holdingABuild(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - if s == nil || s.SentAt != nil || s.Key { - continue - } - if s.State == "built" || (s.State == "asked" && s.Build != "") { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// resumeStopped sets a walk stopped by hand building again at the tier it stopped at, its stop cleared and its asks kept -// (issue 1886): the next look carries it on as any walk — a build done waits for its reference, and one done on the -// builder's key is handed on (novox/hq ADR 0522 §3). Its note keeps who stopped it and that a person resumed it. -func resumeStopped(p *inventory.Plan) string { - asked, unsent, unasked := waitingIn(*p) - later := 0 - for i := p.Tier + 1; i < len(p.Tiers); i++ { - later += len(p.Tiers[i]) - } - by, why := p.Delivery.Stopped, p.Delivery.StoppedWhy - waits := fmt.Sprintf("%d asked, %d built and not sent, %d not yet asked in it; %d not yet asked in the tiers after", - asked, unsent, unasked, later) - p.State = inventory.PlanBuilding - goesOnAgain(p) - p.Delivery.Kept = nil - p.Note = fmt.Sprintf("tier %d resumed by hand after the stop by %s (%s): %s", p.Tier, by, why, waits) - return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; this verb asks nothing — the next look carries it "+ - "on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits) -} - -// silentGateWords say, in a plan's note, that the gate between two tiers found every machine it waits on silent past -// ApplySilentAfter, and failed the plan as lost (issue 1160, ADR 0531): one text for the writer and the reader (lostToSilence). -const silentGateWords = "every machine it waits on said nothing on its send within" - -// lostToSilence is whether a plan failed only at the gate after its tier, every machine it waited on silent (ADR 0531, -// issue 1886): nothing in its tier failed to build, stopped rolling out or is still to ask — the look had sent the whole tier. -func lostToSilence(p inventory.Plan) bool { - if p.State != inventory.PlanFailed || p.Tier >= len(p.Tiers) || !strings.Contains(p.Note, silentGateWords) || - len(failedIn(p)) > 0 || len(stoppedRollouts(p)) > 0 { - return false - } - asked, _, unasked := waitingIn(p) - return asked == 0 && unasked == 0 -} - -// resumeLost sets a walk lost to silence building again at the tier it failed after (ADR 0531, issue 1886), for the next look to -// judge its gate again on the machines' reports, unless every machine its tier was sent to is still silent: then -// nothing it needs is delivered anywhere yet, and the retry is refused as the gate would fail it again. -func resumeLost(p *inventory.Plan, silent map[string]bool) (string, error) { - var sent, still []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil { - for n := range s.Rest { - if !slices.Contains(sent, n) { - sent = append(sent, n) - } - } - } - } - for _, n := range sent { - if silent[n] { - still = append(still, n) - } - } - sort.Strings(still) - if len(sent) > 0 && len(still) == len(sent) { - return "", fmt.Errorf("%s failed because every machine its tier %d waits on said nothing on its send, and %s "+ - "still said nothing: retry once it reported", p.ID, p.Tier+1, strings.Join(still, ", ")) - } - was := p.Note - p.State = inventory.PlanBuilding - goesOnAgain(p) - p.Note = fmt.Sprintf("tier %d resumed by hand after it failed as lost (%s): its gate is judged again on the "+ - "machines' reports", p.Tier, was) - return fmt.Sprintf("%s resumed at tier %d of %d, failed as lost to silence (%s); this verb asks and sends nothing — "+ - "the next look judges its gate again and carries it on as any walk", p.ID, p.Tier+1, len(p.Tiers), was), nil -} diff --git a/cmd/mesh-controller/plan_retry_stopped_test.go b/cmd/mesh-controller/plan_retry_stopped_test.go deleted file mode 100644 index a4552dfd..00000000 --- a/cmd/mesh-controller/plan_retry_stopped_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk stopped by hand while its tier waited — modules asked on the builder's keys, two built and never sent, the next -// tier not yet asked — is resumed by `plans retry` (issue 1886): set building again, its stop cleared, the asks it holds -// kept, so the next look carries it on as any walk. Nothing in it failed, so the retry of failed builds had nothing to do -// and refused it, leaving a stopped R4e with no verb to carry its 48 modules on. -func TestRetryResumesAWalkStoppedByHand(t *testing.T) { - at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC) - sent := at.Add(-time.Hour) - p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16", - Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1, - Note: "stopped by a person at tier 1: issue 1886", - Delivery: &inventory.PlanDelivery{Stopped: "a person (fix-1886@shanks)", StoppedWhy: "issue 1886", - Kept: []inventory.PlanKept{{Module: "mesh-controller"}}}, - Modules: map[string]*inventory.PlanModule{}} - tier0 := []string{"mesh-controller", "mesh-sdk", "pacman", "route-proxy"} - for _, m := range tier0 { - p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent} - } - var tier1 []string - for _, m := range []string{"mesh-tools-go", "mesh-tools"} { - tier1 = append(tier1, m) - p.Modules[m] = &inventory.PlanModule{State: "built", AskedAt: &sent, BuiltAt: &sent, Build: "build-" + m} - } - onKeys := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node", "mesh-delivery-checker", - "mesh-delivery-forge-bridge", "mesh-delivery-gate", "mesh-delivery-indexer", "mesh-delivery-view"} - for _, m := range onKeys { - tier1 = append(tier1, m) - p.Modules[m] = &inventory.PlanModule{State: "asked", AskedAt: &sent, Key: true, Build: "build-" + m} - } - tier2 := []string{"gitea", "mesh-catalog", "mongodb"} - for _, m := range tier2 { - p.Modules[m] = &inventory.PlanModule{} - } - p.Tiers = [][]string{tier0, tier1, tier2} - - if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil { - t.Fatalf("a walk stopped by hand with its tier waiting was refused a retry: %v", err) - } - said := resumeStopped(&p) - if p.State != inventory.PlanBuilding { - t.Errorf("resumed walk is %s, wanted building", p.State) - } - if p.Delivery.Stopped != "" || p.Delivery.StoppedWhy != "" || len(p.Delivery.Kept) > 0 { - t.Errorf("resumed walk still says it stopped: %+v", p.Delivery) - } - if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(p.Note, "a person (fix-1886@shanks)") { - t.Errorf("the note does not record the hand resume and the stop it undid: %q", p.Note) - } - for _, m := range onKeys { - s := p.Modules[m] - if s.State != "asked" || !s.Key || s.Build != "build-"+m { - t.Errorf("%s's ask was not kept for the look to hand on: %+v", m, s) - } - } - for _, w := range []string{"plan-r4e", "tier 2 of 3", "8 asked", "2 built and not sent", "0 not yet asked in it", - "3 not yet asked in the tiers after"} { - if !strings.Contains(said, w) { - t.Errorf("the answer %q does not say %q", said, w) - } - } - - // What it still refuses: a walk stopped with nothing left in its tier, and one a related open walk runs beside. - done := p - done.State, done.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} - done.Modules = map[string]*inventory.PlanModule{} - for _, tier := range done.Tiers { - for _, m := range tier { - done.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent} - } - } - if err := retryRefusal(done, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 1") { - t.Errorf("a stopped walk with nothing left in its tier was retried: %v", err) - } - stopped := p - stopped.State, stopped.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} - beside := inventory.Plan{ID: "plan-442", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a", - Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}}, - Modules: map[string]*inventory.PlanModule{"mesh-sdk": {State: "asked"}}} - if err := retryRefusal(stopped, []inventory.Plan{beside}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") { - t.Errorf("a stopped walk was resumed beside a related open walk: %v", err) - } - // A newer open plan of its repository and branch supersedes it: resuming would build what that one replaced. - newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed", - Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}}, - Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}} - if err := retryRefusal(stopped, []inventory.Plan{stopped, newerOpen}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "plan-newer supersedes it") { - t.Errorf("a stopped walk was resumed under a newer open plan of its branch: %v", err) - } - // A newer plan holding a build of its tier not yet sent refuses it: resuming would send the older build over the newer. - newerBuilt := inventory.Plan{ID: "plan-agent", Repository: "novox/mesh-controller", Branch: "main", Commit: "d00dfeed", - Created: at, State: inventory.PlanDone, Tiers: [][]string{{"mesh-tools"}}, - Modules: map[string]*inventory.PlanModule{"mesh-tools": {State: "built"}}} - if err := retryRefusal(stopped, []inventory.Plan{stopped, newerBuilt}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "older build") { - t.Errorf("a stopped walk was resumed over a newer plan of a build it has not sent: %v", err) - } -} - -// A newer plan holding a module the stopped walk has not yet asked does not refuse its resume (review of issue 1886): that -// module is asked at its followed branch's head when the walk reaches it, so no older build goes over the newer. Open and -// related, the newer walk refuses it under one walk at a time; done, it does not. One holding a build the walk has not sent -// still refuses it. -func TestResumeIsNotRefusedByANewerPlanOfAModuleNotYetAsked(t *testing.T) { - at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC) - sent := at.Add(-time.Hour) - p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16", - Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1, - Delivery: &inventory.PlanDelivery{Stopped: "a person", StoppedWhy: "issue 1886"}, - Tiers: [][]string{{"mesh-sdk"}, {"build-agent", "mesh-delivery-gate"}, {"mesh-delivery-builder", "gitea"}}, - Modules: map[string]*inventory.PlanModule{ - "mesh-sdk": {State: "built", BuiltAt: &sent, SentAt: &sent}, - "build-agent": {State: "built", BuiltAt: &sent}, - "mesh-delivery-gate": {State: "asked", AskedAt: &sent, Key: true}, - "mesh-delivery-builder": {}, "gitea": {}}} - newer := func(state string, holds string) inventory.Plan { - return inventory.Plan{ID: "plan-1792715180015327197", Repository: "novox/mesh-delivery", Branch: "main", - Commit: "d00dfeed", Created: at, State: state, Tiers: [][]string{{holds}}, - Modules: map[string]*inventory.PlanModule{holds: {State: "asked"}}} - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-builder")}, ownPathOf(nil)); err != nil { - t.Errorf("a done newer plan of a module not yet asked refused the resume: %v", err) - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-gate")}, ownPathOf(nil)); err != nil { - t.Errorf("a done newer plan of a module on the builder's key refused the resume: %v", err) - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanBuilding, "mesh-delivery-builder")}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "one walk at a time") { - t.Errorf("an open newer walk related to it did not refuse the resume under one walk at a time: %v", err) - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "build-agent")}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "older build") { - t.Errorf("a newer plan of a built module not yet sent did not refuse the resume: %v", err) - } -} diff --git a/cmd/mesh-controller/plan_retry_tier_test.go b/cmd/mesh-controller/plan_retry_tier_test.go deleted file mode 100644 index 7efd44a8..00000000 --- a/cmd/mesh-controller/plan_retry_tier_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea -// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan, -// and plex's, arriving after, found no open plan and was kept only in the build records. The first -// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked -// plex. -func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) { - asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) - failedAt := asked.Add(2 * time.Minute) - p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, - Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{ - "gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"}, - "plex": {State: "asked", AskedAt: &asked, Build: "build-plex"}, - }} - byID := map[string]inventory.Build{ - "build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"}, - } - - if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) { - t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got) - } -} - -// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's -// once the retry sets it building, and one that is recorded built is taken as built. -func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) { - asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) - builtAt := asked.Add(3 * time.Minute) - p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, - Tiers: [][]string{{"a", "b", "c"}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"}, - "b": {State: "asked", AskedAt: &asked, Build: "build-b"}, - "c": {State: "asked", AskedAt: &asked, Build: "build-c"}, - }} - byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}} - - if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) { - t.Fatalf("asks %v, want a alone", got) - } - if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" { - t.Errorf("c, recorded built, is %+v", s) - } - if s := p.Modules["b"]; s.State != "asked" { - t.Errorf("b, still building, is %+v", s) - } -} diff --git a/cmd/mesh-controller/plan_test.go b/cmd/mesh-controller/plan_test.go index a9e0df65..221e9231 100644 --- a/cmd/mesh-controller/plan_test.go +++ b/cmd/mesh-controller/plan_test.go @@ -1,51 +1,3 @@ package main -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// `plan` tells a person what a module would open and why, from the same `why` every listens -// entry already carries for the firewall (novox/hq ADR 0007) — so deciding whether to assign a -// module does not need reading its manifest first. -func TestListensLinesShowWhatAModuleWouldOpenAndWhy(t *testing.T) { - m := catalogue.Manifest{Module: "minio", Listens: []catalogue.Listening{ - {Port: 9000, From: catalogue.FromMesh, Why: "the S3 endpoint"}, - {Port: 9001, From: catalogue.FromMesh}, - }} - got := listensLines(m) - if len(got) != 2 { - t.Fatalf("two listens entries, got %d: %v", len(got), got) - } - if !strings.Contains(got[0], "9000/tcp") || !strings.Contains(got[0], "the S3 endpoint") { - t.Errorf("the port and its why did not both appear: %q", got[0]) - } - if strings.Contains(got[1], "—") { - t.Errorf("a listens entry with no why should not print a dash: %q", got[1]) - } - if !strings.Contains(got[1], "9001/tcp") { - t.Errorf("the port still appears without a why: %q", got[1]) - } -} - -func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) { - if got := listensLines(catalogue.Manifest{Module: "board"}); len(got) != 0 { - t.Errorf("a module with no listens should print nothing, got %v", got) - } -} - -// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the -// mesh's region of a hosts file reads as the whole file. -func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) { - region := shownAs(map[string]any{ - "id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"}) - if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" { - t.Errorf("the region is shown as %q", region) - } - whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"}) - if strings.Contains(whole, "written into") { - t.Errorf("a whole file is shown as written into: %q", whole) - } -} +import () diff --git a/cmd/mesh-controller/planner_records_test.go b/cmd/mesh-controller/planner_records_test.go deleted file mode 100644 index 209d0710..00000000 --- a/cmd/mesh-controller/planner_records_test.go +++ /dev/null @@ -1,153 +0,0 @@ -//go:build integration - -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **The planner over edges the store derives**, not edges written by hand: modules registered, builds -// recorded with what they stood on and which repositories they read, the relation answered by -// inventory.Dependencies (dependenciesOf over the records), and the merge planned by reachOfMerge — the -// path a real merge takes, short of the bus. -// -// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267, -// issues 338 and 363): each build records the build source it said, and a merge is mapped onto those of the -// newest builds. A build that said none (P below, as every build before ADR 0267) is read as before: P moves -// on any merge to the repository it packages, though through no edge. The rows tagged 338 held the opposite -// until ADR 0267 was built. -func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) { - inv := inventory.ForTest(t) - ctx := t.Context() - asked := time.Now().Add(-time.Hour) - sourcesOf := map[string][]inventory.BuildSource{} - register := func(m catalogue.Manifest, repository, path string, against []string, read []inventory.ReadRepository) { - t.Helper() - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path, - Ref: "main", BuiltFrom: "old", Asked: asked}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main", - Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked, - Sources: sourcesOf[m.Module]}); err != nil { - t.Fatal(err) - } - } - controllerRead := []inventory.ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}} - agent := catalogue.Manifest{Module: "build-agent", Version: "1", - Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}} - - // The shape of issue 338, each build saying its build source (ADR 0267). - gomod := []string{"go.mod", "go.sum"} - sourcesOf["mesh-controller"] = []inventory.BuildSource{{Paths: append([]string{"module.json", "cmd/mesh-controller/", - "internal/conditions/", "internal/broker/"}, gomod...)}} - sourcesOf["build-agent"] = []inventory.BuildSource{ - {Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}}, - {Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"cmd/mesh-builder/", "internal/broker/"}, gomod...)}} - sourcesOf["route-proxy"] = []inventory.BuildSource{ - {Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}}, - {Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"modules/route-proxy/cmd/route-proxy/", "internal/broker/"}, gomod...)}} - register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil) - register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead) - register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead) - register(catalogue.Manifest{Module: "gitea", Version: "1"}, "novox/mesh-catalog", "modules/gitea", nil, nil) - // A, B and C in one repository; D built against A's artifact, E declaring B, P packaging the repository. - for _, n := range []string{"a", "b", "c"} { - register(catalogue.Manifest{Module: n, Version: "1"}, "novox/one", "modules/"+n, nil, nil) - } - register(catalogue.Manifest{Module: "d", Version: "1"}, "novox/two", "d", - []string{catalogue.ArtifactStoreScheme + "a/runtime@sha256:" + strings.Repeat("0", 64)}, nil) - register(catalogue.Manifest{Module: "e", Version: "1", Build: &catalogue.Build{ - On: []catalogue.BuildsOn{{Arg: "BASE", Module: "b", Artifact: "runtime"}}}}, "novox/two", "e", nil, nil) - register(catalogue.Manifest{Module: "p", Version: "1"}, "novox/two", "p", nil, - []inventory.ReadRepository{{Repository: "novox/one", Ref: "main"}}) - - entries, err := inv.Catalogued(ctx) - if err != nil { - t.Fatal(err) - } - read, err := inv.ReadRepositories(ctx) - if err != nil { - t.Fatal(err) - } - edges, err := inv.Dependencies(ctx) - if err != nil { - t.Fatal(err) - } - - // The edges the hand-written rows of TestASharedRepositoryMovesWhatPackagesItAsItDoesToday use are - // the ones derived here. - var shared []inventory.Edge - in338 := map[string]bool{"mesh-controller": true, "build-agent": true, "route-proxy": true, "gitea": true} - for _, e := range edges { - if in338[e.From] && in338[e.To] { - shared = append(shared, e) - } - } - if !reflect.DeepEqual(shared, sharedRepositoryEdges) { - t.Errorf("derived %v\nthe hand-written rows use %v", shared, sharedRepositoryEdges) - } - // Each kind derived from its record: built against (stands-on), build.on (declared); a read draws none. - for _, e := range edges { - if e.Kind == inventory.EdgePackages { - t.Errorf("a packages edge was drawn (ADR 0267 rule 4): %v", e) - } - } - for _, want := range []inventory.Edge{ - dep("d", inventory.EdgeStandsOn, "a"), - dep("e", inventory.EdgeDeclared, "b"), - dep("d", inventory.EdgeBuiltBy, "build-agent"), - } { - found := false - for _, e := range edges { - found = found || e == want - } - if !found { - t.Errorf("no %s %s %s derived: %v", want.From, want.Kind, want.To, edges) - } - } - - for _, c := range []struct { - what, repo string - paths []string - want string - issue338 bool - }{ - {"A and B changed, C untouched: D after A, E after B; P, which said no build source, reads all", "one", - []string{"modules/a/x.go", "modules/b/x.go"}, "a,b,p | d,e", false}, - {"C alone: C, and P, read whole as before; P after nothing", "one", - []string{"modules/c/x.go"}, "c,p", false}, - {"a README of the repository P packages: P, read whole as before", "one", - []string{"README.md"}, "p", false}, - {"the dependent's repository: D alone", "two", []string{"d/main.go"}, "d", false}, - {"a README of the controller's repository: no module", "mesh-controller", - []string{"README.md"}, "", true}, - {"the controller's own command: the controller alone", "mesh-controller", - []string{"cmd/mesh-controller/main.go"}, "mesh-controller", true}, - {"a package only the controller builds from: the controller alone", "mesh-controller", - []string{"internal/conditions/condition.go"}, "mesh-controller", true}, - {"the route proxy's program: the route proxy alone", "mesh-controller", - []string{"modules/route-proxy/cmd/route-proxy/main.go"}, "route-proxy", true}, - {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false}, - {"the route proxy's directory in the catalogue: it alone", "mesh-catalog", - []string{"modules/route-proxy/module.json"}, "route-proxy", false}, - {"the build agent's directory: it alone, nothing it builds", "mesh-catalog", - []string{"modules/build-agent/module.json"}, "build-agent", false}, - } { - _, got := planMerge(t, c.repo, c.paths, entries, read, edges) - if got != c.want { - tag := "" - if c.issue338 { - tag = " (issue 338, flipped by ADR 0267)" - } - t.Errorf("%s: planned %q, wanted %q%s", c.what, got, c.want, tag) - } - } -} diff --git a/cmd/mesh-controller/planner_rules_test.go b/cmd/mesh-controller/planner_rules_test.go deleted file mode 100644 index 4e14a594..00000000 --- a/cmd/mesh-controller/planner_rules_test.go +++ /dev/null @@ -1,672 +0,0 @@ -package main - -import ( - "fmt" - "math/rand/v2" - "sort" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The delivery planner's rules, as recorded (novox/hq ADR 0162 §1, ADR 0238 §3), held by one table and -// one property over reachOfMerge — the planner's one answer to "what does this merge move, and in which -// order". A row that fails here on main is a planner that breaks a recorded rule: the row stays, the -// expectation is not bent to the code. -// -// The kinds of edge, as the code reads them (release_plan.go): -// -// kind widens the plan orders the tiers -// stands-on yes yes, after its base is built -// declared yes yes, after its base is built -// packages no no — retired by novox/hq ADR 0267; one recorded before is read and ignored -// built-by no yes, after the build machine — except for what the build machine stands -// on, and for the controller whose worker it binds -// worker-of no yes, the build seat's holder after the controller (hq issue 206) - -const ( - repoOne = "http://forge.internal:20000/novox/one.git" - repoTwo = "http://forge.internal:20000/novox/two.git" -) - -// dep is one edge of the catalogue's relation: from depends on to, in the way kind says. -func dep(from, kind, to string) inventory.Edge { - return inventory.Edge{From: from, To: to, Kind: kind} -} - -// tiered is a plan's tiers as one line: a tier's modules by comma, tiers by " | ". Empty for no plan. -func tiered(tiers [][]string) string { - var out []string - for _, t := range tiers { - out = append(out, strings.Join(t, ",")) - } - return strings.Join(out, " | ") -} - -// planMerge is what reachOfMerge plans for a merge of these files into a repository's main: its tiers as -// one line, and the files no build reads. It also holds the plan to its own shape: every module it builds -// is in exactly one tier. -func planMerge(t *testing.T, repo string, paths []string, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, edges []inventory.Edge) (mergeReach, string) { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: "head", Paths: paths} - r := reachOfMerge(m, entries, read, edges) - seen := map[string]int{} - for _, tier := range r.Plan.Tiers { - for _, name := range tier { - seen[name]++ - } - } - for name := range r.Plan.Modules { - if seen[name] != 1 { - t.Errorf("%s/%v: %s is in %d tiers of %v", repo, paths, name, seen[name], r.Plan.Tiers) - } - } - if len(seen) != len(r.Plan.Modules) { - t.Errorf("%s/%v: the tiers %v hold modules the plan does not (%d)", repo, paths, r.Plan.Tiers, len(r.Plan.Modules)) - } - return r, tiered(r.Plan.Tiers) -} - -// **A merge moves the modules whose directory it changed, and everything built on them; nothing else.** -// Each row is a catalogue (modules in directories of one or two repositories), its dependencies with -// their kinds, the files one commit changed, and the exact plan: the moved set and its tier order. -func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) { - in := func(repo, dir string, names ...string) []inventory.Entry { - var out []inventory.Entry - for _, n := range names { - d := dir + "/" + n - if dir == "" { - d = n - } - out = append(out, fromRepo(n, repo, d)) - } - return out - } - // Modules a to f, x and z in novox/one under modules/; g in novox/two at g/. - entries := append(in(repoOne, "modules", "a", "b", "c", "d", "e", "f", "x", "z"), in(repoTwo, "", "g")...) - const ( - standsOn = inventory.EdgeStandsOn - declared = inventory.EdgeDeclared - packages = inventory.EdgePackages - builtBy = inventory.EdgeBuiltBy - workerOf = inventory.EdgeWorkerOf - ) - // The two real cycles the kinds resolve themselves (ADR 0162 §1, hq issue 206). - runtime := append(in(repoOne, "modules", "runtime", "builder"), fromRepo("controller", repoTwo, "")) - for _, c := range []struct { - what string - entries []inventory.Entry - edges []inventory.Edge - repo string - paths []string - want string // the tiers, " | " between them - unread string - cycle bool - }{ - // The operator's case: two modules changed, a third beside them in the same repository untouched, - // each changed one with a dependent. - {what: "A and B changed, C untouched beside them, D on A and E on B", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")}, - repo: "one", paths: []string{"modules/a/main.go", "modules/b/module.json"}, want: "a,b | d,e"}, - {what: "only A's directory: A and what stands on it", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")}, - repo: "one", paths: []string{"modules/a/main.go"}, want: "a | d"}, - {what: "only C's directory: C alone, nothing is built on it", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")}, - repo: "one", paths: []string{"modules/c/Dockerfile"}, want: "c"}, - {what: "only the dependent changed: its base does not move", - edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/d/main.go"}, want: "d"}, - {what: "transitive: F on D on A, A changed", - edges: []inventory.Edge{dep("f", standsOn, "d"), dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | d | f"}, - {what: "transitive across kinds stops at a packages edge: F declared on D, D packages A (ADR 0267)", - edges: []inventory.Edge{dep("f", declared, "d"), dep("d", packages, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - - // Each kind alone: X depends on A, A changed (widening), then both changed (ordering). - {what: "stands-on (built against A's artifact) widens", edges: []inventory.Edge{dep("x", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | x"}, - {what: "stands-on orders", edges: []inventory.Edge{dep("x", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, - {what: "declared (build.on) widens", edges: []inventory.Edge{dep("x", declared, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | x"}, - {what: "declared orders", edges: []inventory.Edge{dep("x", declared, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, - {what: "packages, recorded before ADR 0267, widens nothing", edges: []inventory.Edge{dep("x", packages, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - {what: "packages does not order", edges: []inventory.Edge{dep("x", packages, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, - {what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - // Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered. - {what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, - {what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - {what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, - {what: "a change to the base alone does not move what it builds", edges: []inventory.Edge{dep("x", builtBy, "a"), - dep("d", builtBy, "a"), dep("e", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/module.json"}, want: "a | e"}, - - // The cycles the kinds resolve: the build machine stands on the runtime image the runtime image is - // built by; the build seat's holder follows the controller that is built by it. - {what: "the build machine's base comes first, built by the build machine that runs", entries: runtime, - edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")}, - repo: "one", paths: []string{"modules/runtime/Dockerfile", "modules/builder/main.go"}, want: "runtime | builder"}, - {what: "the runtime image alone takes the build machine on it along", entries: runtime, - edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")}, - repo: "one", paths: []string{"modules/runtime/Dockerfile"}, want: "runtime | builder"}, - {what: "the build machine alone moves alone", entries: runtime, - edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")}, - repo: "one", paths: []string{"modules/builder/main.go"}, want: "builder"}, - {what: "the build seat's holder follows the controller it binds the worker of", entries: runtime, - edges: []inventory.Edge{dep("controller", builtBy, "builder"), dep("builder", workerOf, "controller")}, - repo: "two", paths: []string{"cmd/main.go"}, want: "controller"}, - - // Two repositories. - {what: "a dependent in another repository follows its base", - edges: []inventory.Edge{dep("g", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | g"}, - {what: "a directory of the same name in another repository is not this one's", - edges: []inventory.Edge{dep("g", standsOn, "a")}, - repo: "two", paths: []string{"modules/a/x"}, want: "", unread: "modules/a/x"}, - {what: "the dependent's own repository moves the dependent alone", - edges: []inventory.Edge{dep("g", standsOn, "a")}, - repo: "two", paths: []string{"g/main.go"}, want: "g"}, - - // A diamond. - {what: "a diamond, one side changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | d"}, - {what: "a diamond, both sides changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")}, - repo: "one", paths: []string{"modules/a/x", "modules/b/x"}, want: "a,b | d"}, - {what: "a diamond on one base", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", declared, "b"), - dep("a", standsOn, "z"), dep("b", standsOn, "z")}, - repo: "one", paths: []string{"modules/z/x"}, want: "z | a,b | d"}, - {what: "a diamond of mixed kinds orders on the ordering side only", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", packages, "b")}, - repo: "one", paths: []string{"modules/b/x"}, want: "b"}, - - // A cycle the catalogue should never produce: what remains is one last tier, and said. - {what: "a cycle is one last tier, not lost", edges: []inventory.Edge{dep("a", standsOn, "b"), dep("b", standsOn, "a"), - dep("f", standsOn, "c")}, - repo: "one", paths: []string{"modules/a/x", "modules/c/x"}, want: "c | f | a,b", cycle: true}, - - // Files no build reads. - {what: "a README at the root of a repository whose modules all live below it", - edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"README.md"}, want: "", unread: "README.md"}, - {what: "a directory no module lives in", edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/lib/x.go", "modules/README.md"}, want: "", - unread: "modules/lib/x.go,modules/README.md"}, - {what: "a module's directory beside a root file", edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"merge-check.sh", "modules/a/x"}, want: "a | d", unread: "merge-check.sh"}, - {what: "a directory whose name begins with a module's", edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/ab/x"}, want: "", unread: "modules/ab/x"}, - } { - e := entries - if c.entries != nil { - e = c.entries - } - r, got := planMerge(t, c.repo, c.paths, e, nil, c.edges) - if got != c.want { - t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want) - } - if u := strings.Join(r.Unread, ","); u != c.unread { - t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread) - } - // A packages edge in the last tier is no cycle; hasCycle said one on main at 8170fc5. - if hasCycle(r.Plan.Tiers, c.edges) != c.cycle { - t.Errorf("%s: a cycle said %v, wanted %v (%v)", c.what, !c.cycle, c.cycle, r.Plan.Tiers) - } - } -} - -// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267, -// issue 338, issue 363). The controller is built from its repository's root as a Go bundle; the route proxy -// and the build seat's holder build images whose context is that repository and which name the package they -// compile. Each newest trunk build said its build source — the import closure of its program — and a merge -// is mapped onto those. The rows tagged 338 held the opposite until ADR 0267 was built: every merge to the -// controller's repository planned all three, in three tiers. -// -// The build sources are this repository's own programs as GoBuildSource reads them (held to that by -// TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn in internal/builder), cut to what the rows need. -func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { - const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git" - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - entries := []inventory.Entry{ - fromRepo("mesh-controller", controllerRepo, ""), - fromRepo("build-agent", catalogueRepo, "modules/build-agent"), - fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"), - fromRepo("gitea", catalogueRepo, "modules/gitea"), - } - gomod := []string{"go.mod", "go.sum", "vendor/modules.txt"} - with := func(paths ...string) []string { return append(append([]string{}, gomod...), paths...) } - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: with("module.json", "cmd/mesh-controller/", "internal/conditions/", - "internal/broker/", "internal/builder/", "internal/inventory/", "internal/inventory/migrations/**", - "vendor/github.com/nats-io/nats.go/")}}, - "build-agent": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: with("cmd/mesh-builder/", "internal/broker/", - "internal/builder/", "internal/inventory/", "internal/inventory/migrations/**", "vendor/github.com/nats-io/nats.go/")}, - {Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}}, - }, - "route-proxy": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: with("modules/route-proxy/cmd/route-proxy/", "internal/broker/", - "vendor/github.com/nats-io/nats.go/")}, - {Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}}, - }, - } - // With no build source said — before each module's first trunk build under ADR 0267, or while an - // earlier merge's build of it is pending — a module is read as before. - unsaid := map[string][]inventory.ReadRepository{ - "build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}}, - "route-proxy": {{Repository: "novox/mesh-controller", Ref: "main"}}, - } - edges := sharedRepositoryEdges - for _, c := range []struct { - what, repo string - paths []string - read map[string][]inventory.ReadRepository - want string - unread string - }{ - // The operator's acceptance: a merge of the controller's own code plans the controller alone. - {"the controller's own command: the controller alone (338)", "mesh-controller", - []string{"cmd/mesh-controller/main.go"}, read, "mesh-controller", ""}, - {"a package only the controller builds from: the controller alone (338)", "mesh-controller", - []string{"internal/conditions/condition.go", "internal/conditions/bus.go"}, read, "mesh-controller", ""}, - {"a test beside the controller's command: nothing is built from it", "mesh-controller", - []string{"cmd/mesh-controller/main_test.go"}, read, "", "cmd/mesh-controller/main_test.go"}, - {"a README of the controller's repository: no module (338)", "mesh-controller", - []string{"README.md"}, read, "", "README.md"}, - {"the route proxy's program alone: the route proxy alone (338)", "mesh-controller", - []string{"modules/route-proxy/cmd/route-proxy/main.go"}, read, "route-proxy", ""}, - {"the build seat's program alone: its holder alone", "mesh-controller", - []string{"cmd/mesh-builder/main.go"}, read, "build-agent", ""}, - {"a package the build seat's program and the controller build from: both", "mesh-controller", - []string{"internal/builder/builder.go"}, read, "mesh-controller | build-agent", ""}, - {"a migration the controller and the build seat's program embed: both", "mesh-controller", - []string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read, - "mesh-controller | build-agent", ""}, - // A package all three build from: all three; the build seat's holder after the controller whose worker - // it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3). - {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, - {"a vendored package all three build from: all three", "mesh-controller", - []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, - {"go.sum: all three", "mesh-controller", - []string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""}, - {"a file added to the proxy's package: the proxy", "mesh-controller", - []string{"modules/route-proxy/cmd/route-proxy/new.go"}, read, "route-proxy", ""}, - // Before any build source is said: as before. - {"no build source said: a README moves all three, as before", "mesh-controller", - []string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""}, - // In the catalogue, where they live, each by its own build source. - {"the route proxy's recipe: it alone", "mesh-catalog", - []string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""}, - {"the route proxy's manifest: it alone", "mesh-catalog", - []string{"modules/route-proxy/module.json"}, read, "route-proxy", ""}, - {"the route proxy's README: nothing", "mesh-catalog", - []string{"modules/route-proxy/README.md"}, read, "", "modules/route-proxy/README.md"}, - {"the build agent's manifest: it alone, nothing it builds", "mesh-catalog", - []string{"modules/build-agent/module.json"}, read, "build-agent", ""}, - {"another module of the catalogue: neither", "mesh-catalog", - []string{"modules/gitea/index.ts"}, read, "gitea", ""}, - } { - r, got := planMerge(t, c.repo, c.paths, entries, c.read, edges) - if got != c.want { - t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want) - } - if u := strings.Join(r.Unread, ","); u != c.unread { - t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread) - } - } - // Files not all said: everything the repository builds, as before. - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head", - Paths: []string{"README.md"}, PathsTruncated: true} - if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" { - t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got) - } -} - -// **A module whose recorded build source a plan has overtaken is read whole** (novox/hq ADR 0267): a merge -// that added an import to the route proxy is planned; before a build of it works — still building, failed, -// or its plan closed before reaching it — a merge changing only the newly imported package must still move -// the proxy, since the build source its last build said does not hold that package. -func TestAModuleAPlanOvertookIsReadWhole(t *testing.T) { - built := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC) - read := map[string][]inventory.ReadRepository{ - "route-proxy": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "go.mod"}, Built: built}, - {Own: true, Paths: []string{"modules/route-proxy/module.json"}, Built: built}, - }, - "mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: built}}, - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Paths: []string{"internal/newly/imported.go"}} - if readsFrom(read["route-proxy"], m) { - t.Fatal("the said build source holds the new package: the fixture is wrong") - } - proxy := func(state string) map[string]*inventory.PlanModule { - return map[string]*inventory.PlanModule{"route-proxy": {State: state}, "gitea": {State: "built"}} - } - for _, c := range []struct { - what string - plans []inventory.Plan - whole bool - }{ - {"no plan", nil, false}, - {"a plan still building it", []inventory.Plan{{State: inventory.PlanBuilding, Created: built.Add(-time.Hour), - Modules: proxy("building")}}, true}, - {"a plan made after its build that failed before building it", []inventory.Plan{{State: "failed", - Created: built.Add(time.Minute), Modules: proxy("waiting")}}, true}, - {"a plan made after its build that built it", []inventory.Plan{{State: inventory.PlanDone, - Created: built.Add(time.Minute), Modules: proxy("built")}}, false}, - {"a plan closed before its build", []inventory.Plan{{State: "failed", Created: built.Add(-time.Hour), - Modules: proxy("waiting")}}, false}, - } { - view := planningView(read, c.plans) - if readsFrom(view["route-proxy"], m) != c.whole { - t.Errorf("%s: read whole %v, wanted %v", c.what, !c.whole, c.whole) - } - if (ownSource(view["route-proxy"]) == nil) != c.whole { - t.Errorf("%s: its own build source kept %v", c.what, ownSource(view["route-proxy"]) != nil) - } - if ownSource(view["mesh-controller"]) == nil { - t.Errorf("%s: a module no plan holds lost its build source", c.what) - } - } -} - -// **A missed merge that moves only a module packaging the repository is acted on** (novox/hq ADR 0267, -// issue 266): the catch-up asks wouldMove, which counts it; once a plan or build of it is made after the -// merge, the merge is history for it, and the catch-up leaves it. -func TestAMissedMergeMovingOnlyAPackagingModuleIsActedOnOnce(t *testing.T) { - merged := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC) - entries := []inventory.Entry{ - fromRepo("mesh-controller", "http://forge.internal:20000/novox/mesh-controller.git", ""), - fromRepo("route-proxy", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/route-proxy"), - } - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: merged.Add(-time.Hour)}}, - "route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/"}, - Built: merged.Add(-time.Hour), Looked: merged.Add(-time.Hour)}}, - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "c1", - MergedAt: merged.Format(time.RFC3339), Paths: []string{"modules/route-proxy/cmd/route-proxy/main.go"}} - if got := wouldMove(m, entries, planningView(read, nil)); len(got) != 1 || got[0].Manifest.Module != "route-proxy" { - t.Fatalf("a missed merge of the proxy's program would move %v", got) - } - // Acted on at once: the plan answering this very merge is a look, however close the clocks. - atOnce := []inventory.Plan{{State: inventory.PlanBuilding, Commit: "c1", Created: merged.Add(2 * time.Second), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}} - if got := wouldMove(m, entries, planningView(read, atOnce)); len(got) != 0 { - t.Fatalf("a merge whose own plan holds the proxy would move %v again", got) - } - acted := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(2 * time.Minute), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}} - if got := wouldMove(m, entries, planningView(read, acted)); len(got) != 0 { - t.Fatalf("a merge acted on for the proxy would move %v again", got) - } - // A plan that closed without building it looked at nothing: the merge is still news for it. - closed := []inventory.Plan{{State: "failed", Created: merged.Add(2 * time.Minute), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "waiting"}}}} - if got := wouldMove(m, entries, planningView(read, closed)); len(got) != 1 { - t.Fatalf("a plan that never built the proxy hid the merge from it: %v", got) - } - // A look just before the merge, on clocks a little apart, is no look after it. - skewed := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(30 * time.Second), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}} - if got := wouldMove(m, entries, planningView(read, skewed)); len(got) != 1 { - t.Fatalf("a look within the clocks' margin made the merge history: %v", got) - } -} - -// sharedRepositoryEdges is what dependenciesOf derives for the catalogue of the test above, sorted as it -// sorts them: no packages edge (novox/hq ADR 0267 rule 4). -var sharedRepositoryEdges = []inventory.Edge{ - dep("build-agent", inventory.EdgeWorkerOf, "mesh-controller"), - dep("gitea", inventory.EdgeBuiltBy, "build-agent"), - dep("mesh-controller", inventory.EdgeBuiltBy, "build-agent"), - dep("route-proxy", inventory.EdgeBuiltBy, "build-agent"), -} - -// **The planner's invariant, over random catalogues.** For any catalogue whose dependencies form no cycle -// and any set of changed files in one repository: -// -// - the plan is exactly the modules of that repository whose directory holds a changed file (every file, -// for a module built from the root), and everything reachable from them along stands-on and declared — -// never along packages (novox/hq ADR 0267), built-by or worker-of; -// - every stands-on, declared, built-by and worker-of edge with both ends in the plan has the module -// depended on in an earlier tier; -// - no cycle is said. -// -// Seeded, so a failure is replayed by its seed and case. -func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *testing.T) { - kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages, - inventory.EdgeBuiltBy, inventory.EdgeWorkerOf} - widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true} - // Built-by orders nothing since novox/hq ADR 0360 §3. - orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, - inventory.EdgeWorkerOf: true} - // Directory names drawn from one pool, so two repositories hold directories of the same name, and one - // is a prefix of another. - dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"} - files := []string{"README.md", "merge-check.sh", "lib/z.go", "docs/x.md", "modules/README.md"} - - falseCycles, firstFalseCycle := 0, "" - for _, seed := range []uint64{1, 2, 3, 0x338, 0x162} { - rng := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) - for n := 0; n < 100; n++ { - repos := 1 + rng.IntN(3) - repoName := func(i int) string { return fmt.Sprintf("r%d", i) } - count := 1 + rng.IntN(12) - var entries []inventory.Entry - repoOf, dirOf := map[string]int{}, map[string]string{} - for i := 0; i < count; i++ { - name := fmt.Sprintf("m%02d", i) - repo := rng.IntN(repos) - dir := dirs[rng.IntN(len(dirs))] - if rng.IntN(12) == 0 { - dir = "" // built from the repository's root - } - repoOf[name], dirOf[name] = repo, dir - entries = append(entries, fromRepo(name, "http://forge.internal:20000/novox/"+repoName(repo)+".git", dir)) - } - // A graph with no cycle: a module depends only on modules made before it. - var edges []inventory.Edge - for i := 1; i < count; i++ { - for j := 0; j < i; j++ { - if rng.IntN(4) == 0 { - edges = append(edges, dep(fmt.Sprintf("m%02d", i), kinds[rng.IntN(len(kinds))], fmt.Sprintf("m%02d", j))) - } - } - } - merged := rng.IntN(repos) - var paths []string - for k := 1 + rng.IntN(4); k > 0; k-- { - if rng.IntN(3) == 0 { - paths = append(paths, files[rng.IntN(len(files))]) - } else { - paths = append(paths, dirs[rng.IntN(len(dirs))]+"/f.go") - } - } - - // What the rules say. - want := map[string]bool{} - for _, e := range entries { - name := e.Manifest.Module - if repoOf[name] != merged { - continue - } - for _, p := range paths { - if d := dirOf[name]; d == "" || p == d || strings.HasPrefix(p, d+"/") { - want[name] = true - } - } - } - for grew := true; grew; { - grew = false - for _, e := range edges { - if widens[e.Kind] && want[e.To] && !want[e.From] { - want[e.From], grew = true, true - } - } - } - - r, _ := planMerge(t, repoName(merged), paths, entries, nil, edges) - got := map[string]bool{} - tierOf := map[string]int{} - for i, tier := range r.Plan.Tiers { - for _, name := range tier { - got[name], tierOf[name] = true, i - } - } - replay := func() string { - return fmt.Sprintf("seed %#x case %d: repository %s, files %v\n modules %v\n edges %v\n tiers %v", - seed, n, repoName(merged), paths, describe(entries), edges, r.Plan.Tiers) - } - if !sameSet(got, want) { - t.Fatalf("planned %v, wanted %v\n%s", keys(got), keys(want), replay()) - } - for _, e := range edges { - if orders[e.Kind] && got[e.From] && got[e.To] && tierOf[e.To] >= tierOf[e.From] { - t.Fatalf("%s %s %s, and %s is in tier %d, not before %s's %d\n%s", e.From, e.Kind, e.To, - e.To, tierOf[e.To], e.From, tierOf[e.From], replay()) - } - } - if hasCycle(r.Plan.Tiers, edges) { - falseCycles++ - if firstFalseCycle == "" { - firstFalseCycle = replay() - } - } - } - } - // Said once, after the other invariants have run over every case, so it hides none of them (hasCycle - // counted a packages edge on main at 8170fc5: 19 of these 500 cases). - if falseCycles > 0 { - t.Errorf("a cycle said of a graph with none in %d of 500 cases; "+ - "the first:\n%s", falseCycles, firstFalseCycle) - } -} - -func sameSet(a, b map[string]bool) bool { - if len(a) != len(b) { - return false - } - for k := range a { - if !b[k] { - return false - } - } - return true -} - -func keys(m map[string]bool) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - sort.Strings(out) - return out -} - -func describe(entries []inventory.Entry) []string { - var out []string - for _, e := range entries { - out = append(out, fmt.Sprintf("%s@%s:%q", e.Manifest.Module, - strings.TrimSuffix(strings.TrimPrefix(e.Source.Repository, "http://forge.internal:20000/novox/"), ".git"), - e.Source.Path)) - } - return out -} - -// **A plan says why each module is in it** (novox/hq ADR 0267, issue 363): the files of its build source the -// merge changed, or why it is read whole — never that it packages a repository as though that moved it. -func TestAPlanSaysWhyEachModuleIsInIt(t *testing.T) { - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - controller := fromRepo("mesh-controller", controllerRepo, "") - agent := fromRepo("build-agent", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/build-agent") - gitea := fromRepo("gitea", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/gitea") - built := time.Date(2026, 10, 10, 12, 26, 0, 0, time.UTC) - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/link/"}, Built: built}}, - "build-agent": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"cmd/mesh-builder/", "internal/link/"}, Built: built}, - {Own: true, Paths: []string{"modules/build-agent/module.json"}, Built: built}, - }, - } - merge := func(repo string, paths ...string) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Paths: paths} - } - open := []inventory.Plan{{ID: "plan-1", State: inventory.PlanBuilding, Created: built.Add(time.Minute), - Modules: map[string]*inventory.PlanModule{"build-agent": {State: "asked"}}}} - for _, c := range []struct { - what string - e inventory.Entry - read map[string][]inventory.ReadRepository - m link.SourceMoved - says string - }{ - {"the controller's own closure", controller, read, merge("mesh-controller", "README.md", "internal/link/handacts.go"), - "its build source changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"}, - {"the build seat's closure, through its context", agent, read, merge("mesh-controller", "internal/link/handacts.go"), - "its build source in novox/mesh-controller changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"}, - {"an open plan has yet to build it", agent, planningView(read, open), merge("mesh-controller", "cmd/mesh-controller/main.go"), - "read whole: plan plan-1 has not built it yet, so every file of novox/mesh-controller, which its build context is, is its build source"}, - {"nothing recorded, built from its root", controller, nil, merge("mesh-controller", "README.md"), - "read whole: no build source recorded, so every file of its repository is its build source"}, - {"nothing recorded, in its directory", gitea, nil, merge("mesh-catalog", "modules/gitea/index.ts"), - "read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/index.ts"}, - {"a root manifest is not in a module's directory", gitea, nil, merge("mesh-catalog", "module.json", "modules/gitea/x.ts"), - "read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/x.ts"}, - {"a context on another branch says nothing of this one", agent, map[string][]inventory.ReadRepository{"build-agent": { - {Repository: "novox/mesh-controller", Ref: "release", Paths: []string{"internal/link/"}}, - {Repository: "novox/mesh-controller", Ref: "main"}}}, merge("mesh-controller", "internal/link/handacts.go"), - "read whole: no build source recorded, so every file of novox/mesh-controller, which its build context is, is its build source"}, - {"files not all said", controller, read, link.SourceMoved{Owner: "novox", Repo: "mesh-controller", PathsTruncated: true, - Paths: []string{"x"}}, "read whole: the merge's changed files were not all said"}, - } { - if got := whyMoved(c.e, c.read[c.e.Manifest.Module], c.m); got != c.says { - t.Errorf("%s:\n said %q\n wanted %q", c.what, got, c.says) - } - } -} - -// **A context naming the module's own repository is the module's own build source** (novox/hq ADR 0420 §4, -// issue 1025). A module registered from the controller's repository whose image compiles cmd/x through a -// context naming that same repository is moved by a change to the program, and by none outside its closure. -func TestAContextNamingTheModulesOwnRepositoryIsItsOwnBuildSource(t *testing.T) { - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - entries := []inventory.Entry{fromRepo("mover", controllerRepo, "modules/mover")} - said := []inventory.ReadRepository{ - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "cmd/x/", "internal/y/"}}, - {Own: true, Paths: []string{"modules/mover/module.json"}}, - } - read := map[string][]inventory.ReadRepository{"mover": inventory.OwnRepositoryAsOwn(said, controllerRepo)} - for _, c := range []struct { - what string - paths []string - want string - }{ - {"the program its image compiles", []string{"cmd/x/main.go"}, "mover"}, - {"a package the program imports", []string{"internal/y/y.go"}, "mover"}, - {"its own manifest", []string{"modules/mover/module.json"}, "mover"}, - {"a file outside its closure", []string{"cmd/other/main.go"}, ""}, - } { - if _, got := planMerge(t, "mesh-controller", c.paths, entries, read, nil); got != c.want { - t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want) - } - } - // A context of another repository stays a context read, and a context of this one that said no paths too. - other := []inventory.ReadRepository{{Repository: "novox/mesh-catalog", Paths: []string{"a/"}}, {Repository: "novox/mesh-controller"}} - if got := inventory.OwnRepositoryAsOwn(other, controllerRepo); len(got) != 2 || got[0].Own || got[1].Own { - t.Errorf("a context that is not this repository's with paths became its own: %+v", got) - } -} diff --git a/cmd/mesh-controller/plans_data_test.go b/cmd/mesh-controller/plans_data_test.go deleted file mode 100644 index f7fc58c0..00000000 --- a/cmd/mesh-controller/plans_data_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// flagsOf reads a command line's flags (and its first positional after --what-if) into a map. -func flagsOf(argv []string) map[string]string { - out := map[string]string{} - for i := 0; i < len(argv); i++ { - if !strings.HasPrefix(argv[i], "--") { - continue - } - name := strings.TrimPrefix(argv[i], "--") - if i+1 < len(argv) && !strings.HasPrefix(argv[i+1], "--") { - out[name] = argv[i+1] - i++ - } else { - out[name] = "true" - } - } - return out -} - -// Without the data form the text answer is unchanged (AC2): no data flag, and the data-only arguments are not passed. -func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) { - argv, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a,b"}) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a,b" { - t.Errorf("the text what-if reads %q", got) - } - if _, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"}); err == nil { - t.Error("a head without data was passed over, not refused") - } - argv, _ = argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "data": "false"}) - if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a" { - t.Errorf("data false reads %q", got) - } -} - -func catalogueVerb(name string) (catalogue.Verb, bool) { return controllerVerb(name) } - -func slicesHas(s []string, v string) bool { - for _, x := range s { - if x == v { - return true - } - } - return false -} diff --git a/cmd/mesh-controller/plans_follow.go b/cmd/mesh-controller/plans_follow.go deleted file mode 100644 index 3edf71d9..00000000 --- a/cmd/mesh-controller/plans_follow.go +++ /dev/null @@ -1,159 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strconv" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// Following a plan (novox/hq issue 229): `plans wait ` holds its answer until the plan moves, reaches its -// next tier or ends, or until its bound passes, and says which. Before it, an agent following a rollout had -// `plans` answer once, in prose, and wrote a loop of its own round the mesh's tools to be told the plan moved. -// -// **No wait is longer than a call is answered within** (link.LongestAnswer, inside the console's thirty -// seconds): a longer one would answer "still running" and leave its caller to ask `calls` — the loop again. -// So a caller following a plan to its end asks again; each call answers the moment the plan moves. - -// maxPlanWait is the longest bound a wait takes, and the bound when none is given: what a call is answered -// within, less the time a verb's command takes to start and read the store. -var maxPlanWait = link.LongestAnswer - planWaitStart - -// planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores. -// A store slow enough to push the read and the answer past it degrades as any long call does: the caller is -// answered that the call is still running, and `calls` with its id says what came of it. -const planWaitStart = 5 * time.Second - -// planWaitEvery is how often a wait reads the plan again. A variable so a test reads it more often -// (novox/hq issue 591). -var planWaitEvery = time.Second - -// The things a wait waits until. -var planWaitUntil = []string{"moved", "tier", "done"} - -// planWaitAsked checks what a wait is asked for: until (moved when empty) and a bound (a duration, or a number -// of seconds; maxPlanWait when empty). -func planWaitAsked(until, bound string) (string, time.Duration, error) { - if until == "" { - until = "moved" - } - known := false - for _, u := range planWaitUntil { - known = known || u == until - } - if !known { - return "", 0, fmt.Errorf("a wait is until %s, not %q", strings.Join(planWaitUntil, ", "), until) - } - d := maxPlanWait - if bound = strings.TrimSpace(bound); bound != "" { - var err error - if secs, isNumber := strconv.Atoi(bound); isNumber == nil { - d = time.Duration(secs) * time.Second - } else if d, err = time.ParseDuration(bound); err != nil { - return "", 0, fmt.Errorf("the bound %q is not a duration (such as 20s) or a number of seconds", bound) - } - if d <= 0 || d > maxPlanWait { - return "", 0, fmt.Errorf("the bound is at most %s, the longest a call is answered within; %s was asked "+ - "— to follow a plan longer, ask again: each call answers the moment the plan moves", maxPlanWait, d) - } - } - return until, d, nil -} - -// planEnded is a plan that will not move again: done, failed or superseded. A batch being assembled is not one. -func planEnded(p inventory.Plan) bool { return !p.Open() && !p.Batch() } - -// waitOnPlan reads the plan until what it waits for happens or the bound passes, and answers the plan as last -// read and what ended the wait: moved (any save of it), tier (its tier changed), done (it ended — which ends -// every wait, since nothing comes after it), or bound. Moved and tier are judged against the plan as first read. -func waitOnPlan(ctx context.Context, read func(context.Context) (inventory.Plan, error), until string, - bound, every time.Duration) (inventory.Plan, string, error) { - first, err := read(ctx) - if err != nil { - return inventory.Plan{}, "", err - } - if planEnded(first) { - return first, "done", nil - } - deadline := time.Now().Add(bound) - tick := time.NewTicker(every) - defer tick.Stop() - last := first - for { - select { - case <-ctx.Done(): - return last, "", ctx.Err() - case <-tick.C: - } - if !time.Now().Before(deadline) { - return last, "bound", nil - } - p, err := read(ctx) - if err != nil { - return last, "", err - } - last = p - switch { - case planEnded(p): - return p, "done", nil - case until == "moved" && p.Revision != first.Revision: - return p, "moved", nil - case until == "tier" && p.Tier != first.Tier: - return p, "tier", nil - } - } -} - -// planWaited is what `plans wait` answers: the plan, what ended the wait, and the line a person reads (`text`, -// which the verb answers as its output and not again in the data). -type planWaited struct { - // Came is what ended the wait: moved, tier, done, or bound when nothing did. - Came string `json:"came"` - Until string `json:"until"` - Waited string `json:"waited"` - Plan planStatus `json:"plan"` - // Revision is the plan's as read: a caller asking again with the same plan sees whether it moved since. - Revision int64 `json:"revision"` - Text string `json:"text"` -} - -func planWaitedDoc(p inventory.Plan, st planStatus, until, came string, waited time.Duration, line string) planWaited { - said := map[string]string{ - "moved": "moved", "tier": "reached another tier", "done": "ended", - "bound": "did not " + map[string]string{"moved": "move", "tier": "reach another tier", "done": "end"}[until], - }[came] - text := fmt.Sprintf("%s %s in %s (waited until %s)\n%s — %s\n", p.ID, said, waited.Round(time.Second), until, - p.ID, line) - return planWaited{Came: came, Until: until, Waited: waited.Round(time.Millisecond).String(), Plan: st, - Revision: p.Revision, Text: text} -} - -// planWaitCommand is `plans wait `: the plan followed until it moves, or the bound passes. -func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsked, boundAsked string) error { - until, bound, err := planWaitAsked(untilAsked, boundAsked) - if err != nil { - return err - } - began := time.Now() - p, came, err := waitOnPlan(ctx, func(ctx context.Context) (inventory.Plan, error) { - return inv.PlanByID(ctx, id) - }, until, bound, planWaitEvery) - if err != nil { - return err - } - // Said as `plans ` says its first line, and as `status` says the plan. - now, one := time.Now(), []inventory.Plan{p} - bounds := readTierBounds(ctx, inv, now) - line := planLineSaid(p, now, bounds.of(p.Repository), func(module string) []string { - on, _ := inv.Running(ctx, module) - return on - }, waitsOf(ctx, inv, one)) + heldSaid(p, heldNow(ctx, inv)) - doc := planWaitedDoc(p, planStatuses(one, now, bounds)[0], until, came, now.Sub(began), line) - return json.NewEncoder(os.Stdout).Encode(doc) -} diff --git a/cmd/mesh-controller/plans_follow_integration_test.go b/cmd/mesh-controller/plans_follow_integration_test.go deleted file mode 100644 index 13968d55..00000000 --- a/cmd/mesh-controller/plans_follow_integration_test.go +++ /dev/null @@ -1,62 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - // Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound - // below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough - // for the wait to have read the plan before it. - was := planWaitEvery - planWaitEvery = 50 * time.Millisecond - t.Cleanup(func() { planWaitEvery = was }) - now := time.Now().UTC() - walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229", - Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - moved := make(chan error, 1) - go func() { - time.Sleep(1500 * time.Millisecond) - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err == nil { - p.Tier, p.State = 1, inventory.PlanRolling - err = open.inventory.SavePlan(ctx, &p) - } - moved <- err - }() - began := time.Now() - var got planWaited - out := printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"}) - }) - if err := <-moved; err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(out), &got); err != nil { - t.Fatalf("plans wait did not answer data: %v: %q", err, out) - } - if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling { - t.Fatalf("the wait answered %+v", got) - } - if waited := time.Since(began); waited > 3*time.Second { - t.Errorf("it answered after %s, not when the plan moved", waited) - } - out = printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"}) - }) - if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" { - t.Fatalf("a plan that did not move answered %+v (%v)", got, err) - } -} diff --git a/cmd/mesh-controller/plans_follow_test.go b/cmd/mesh-controller/plans_follow_test.go deleted file mode 100644 index 33b19bc8..00000000 --- a/cmd/mesh-controller/plans_follow_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A plan as the store would answer it at each read: the reads are counted, and each answers the next. -type planReads struct { - plans []inventory.Plan - n int -} - -func (r *planReads) read(context.Context) (inventory.Plan, error) { - p := r.plans[min(r.n, len(r.plans)-1)] - r.n++ - return p, nil -} - -func followedPlan(rev int64, state string, tier int) inventory.Plan { - return inventory.Plan{ID: "plan-1", Repository: "novox/x", Commit: "abc", State: state, Tier: tier, - Tiers: [][]string{{"a"}, {"b"}}, Revision: rev} -} - -// **`plans` waits for a plan, and says what ended the wait** (novox/hq issue 229): an agent following a -// rollout is told when the plan moved, reached its next tier or ended, or that the bound passed — in one -// call, without a loop of its own. -func TestAWaitForAPlanEndsWhenWhatItWaitsForHappens(t *testing.T) { - cases := []struct { - name string - until string - reads []inventory.Plan - came string - times int - }{ - {"moved: any save", "moved", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(1, inventory.PlanBuilding, 0), - followedPlan(2, inventory.PlanBuilding, 0)}, "moved", 3}, - {"tier: a save inside the tier is not enough", "tier", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanBuilding, 0), - followedPlan(3, inventory.PlanRolling, 1)}, "tier", 3}, - {"tier: a plan that ended has no next tier to wait for", "tier", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanFailed, 0)}, "done", 2}, - {"done: a new tier is not the end", "done", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanRolling, 1), - followedPlan(3, inventory.PlanDone, 1)}, "done", 3}, - {"done: a plan already ended answers at once", "done", []inventory.Plan{ - followedPlan(4, inventory.PlanSuperseded, 1)}, "done", 1}, - {"the bound passes", "moved", []inventory.Plan{followedPlan(1, inventory.PlanBuilding, 0)}, "bound", -1}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - r := &planReads{plans: c.reads} - p, came, err := waitOnPlan(context.Background(), r.read, c.until, 50*time.Millisecond, time.Millisecond) - if err != nil { - t.Fatal(err) - } - if came != c.came { - t.Fatalf("the wait ended as %q, want %q", came, c.came) - } - if c.times > 0 && r.n != c.times { - t.Errorf("read the plan %d times, want %d: it answered before or after what it waited for", r.n, c.times) - } - if want := c.reads[min(r.n, len(c.reads))-1]; p.Revision != want.Revision { - t.Errorf("answered revision %d, not the plan as it was last read (%d)", p.Revision, want.Revision) - } - }) - } -} - -// What a wait is asked for is checked before anything is read: what it waits until, and a bound no longer than -// a call is answered within. -func TestAWaitIsRefusedWhatItCannotDo(t *testing.T) { - for _, bad := range []struct{ until, bound string }{{"landed", ""}, {"", "1h"}, {"", "-3s"}, {"", "soon"}} { - if _, _, err := planWaitAsked(bad.until, bad.bound); err == nil { - t.Errorf("until %q, bound %q was taken", bad.until, bad.bound) - } - } - until, bound, err := planWaitAsked("", "") - if err != nil || until != "moved" || bound != maxPlanWait { - t.Fatalf("the defaults are %q %s %v, want moved and %s", until, bound, err, maxPlanWait) - } - if _, bound, _ := planWaitAsked("tier", "12"); bound != 12*time.Second { - t.Errorf("a bare number is seconds; read %s", bound) - } -} - -// **The answer is data, with its prose as a field beside it** (novox/hq issue 229): the verb's `answer` carries -// the plan and what ended the wait, and its output is the line a person reads — never JSON inside a string. -func TestAWaitForAPlanAnswersAsData(t *testing.T) { - p := followedPlan(7, inventory.PlanRolling, 1) - st := planStatuses([]inventory.Plan{p}, time.Now(), tierBounds{})[0] - doc := planWaitedDoc(p, st, "tier", "tier", 3*time.Second, "rolling, tier 2 of 2") - stdout, _ := json.Marshal(doc) - a := answerOf([]string{"plans", "wait", "plan-1"}, stdout, "", true) - data, ok := a.Answer.(map[string]any) - if !ok { - t.Fatalf("the answer is not data: %v", a.Answer) - } - if data["came"] != "tier" || data["revision"] != float64(7) { - t.Errorf("the data does not say what ended the wait: %v", data) - } - if plan, _ := data["plan"].(map[string]any); plan["state"] != "rolling" || plan["tier"] != float64(1) { - t.Errorf("the data does not carry the plan: %v", data["plan"]) - } - if _, text := data["text"]; text { - t.Errorf("the prose is in the data too: %v", data) - } - if strings.Contains(a.Output, "{") || !strings.Contains(a.Output, "plan-1") || !strings.Contains(a.Output, "tier") { - t.Errorf("the output is not the line a person reads: %q", a.Output) - } -} - -// The verb composes the wait, and asks for a window as long as its bound. -func TestThePlansVerbComposesAWait(t *testing.T) { - argv, err := argvFor("plans", map[string]any{"wait": "plan-1", "until": "done", "bound": "15s"}) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(argv, " "); got != "plans wait plan-1 --until done --bound 15s" { - t.Fatalf("composed %q", got) - } - if _, err := argvFor("plans", map[string]any{"until": "done"}); err == nil { - t.Error("until without wait was taken") - } - if _, err := argvFor("plans", map[string]any{"wait": "plan-1", "bound": "1h"}); err == nil { - t.Error("a bound past what a call can wait was taken") - } -} - -// Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as -// data, and says the bound passed when nothing moved (novox/hq issue 229). - -// The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the -// catalogue cannot read maxPlanWait, so this holds them together. -func TestThePlansSchemaSaysTheLongestBoundThereIs(t *testing.T) { - v, ok := controllerVerb("plans") - if !ok { - t.Fatal("no plans verb") - } - props, _ := v.Input["properties"].(map[string]any) - bound, _ := props["bound"].(map[string]any) - said, _ := bound["description"].(string) - if !strings.Contains(said, maxPlanWait.String()+", the default, is the longest") { - t.Fatalf("the schema says %q; the longest bound is %s", said, maxPlanWait) - } -} diff --git a/cmd/mesh-controller/plans_gone_store_integration_test.go b/cmd/mesh-controller/plans_gone_store_integration_test.go new file mode 100644 index 00000000..2e5e71d3 --- /dev/null +++ b/cmd/mesh-controller/plans_gone_store_integration_test.go @@ -0,0 +1,28 @@ +//go:build integration + +package main + +import ( + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// **The controller opens on a store after release 6's migrations, which drop nothing the controller of before reads** (issue +// 1945, review round 1): it assigns a module, reads where it runs and the nodes. Release 6 reads none of the plans' tables. +func TestTheControllerOpensOnTheStoreAfterReleaseSix(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + register(t, open, catalogue.Manifest{Module: "notes", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "notes"); err != nil { + t.Fatalf("assign on the migrated store: %v", err) + } + if on, err := inv.Assigned(ctx, "anchor"); err != nil || !slices.Contains(on, "notes") { + t.Fatalf("assigned %v, want notes among them (an assignment may bring the modules it needs): %v", on, err) + } + if nodes, err := inv.Nodes(ctx); err != nil || len(nodes) == 0 { + t.Fatalf("nodes %v: %v", nodes, err) + } +} diff --git a/cmd/mesh-controller/plans_one_at_a_time_test.go b/cmd/mesh-controller/plans_one_at_a_time_test.go deleted file mode 100644 index 3f3eef97..00000000 --- a/cmd/mesh-controller/plans_one_at_a_time_test.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 213: for the moment a machine hands its controller over, the container and the -// process both run the plan timer on one store. Only the one holding the plans moves them; the other -// leaves them alone, and moves them once they are let go. -func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - now := time.Now().UTC() - // Every tier done: the next step is the plan's last, and needs nothing but the store. - plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now, - State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - // The other controller: its own connections to the same store, holding the plans. - other, err := inventory.Open(ctx) - if err != nil { - t.Fatal(err) - } - t.Cleanup(other.Close) - release, err := other.HoldPlans(ctx, false) - if err != nil { - t.Fatal(err) - } - t.Cleanup(release) // before the close above: a pool waits for a connection still held - - advancePlans(ctx, open) - if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() { - t.Fatalf("a controller moved a plan another held: %+v %v", p, err) - } - - release() - advancePlans(ctx, open) - if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone { - t.Fatalf("the plan did not move once it was let go: %+v %v", p, err) - } -} diff --git a/cmd/mesh-controller/probes.go b/cmd/mesh-controller/probes.go index b082fbb5..2eb138f0 100644 --- a/cmd/mesh-controller/probes.go +++ b/cmd/mesh-controller/probes.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "net" "regexp" "slices" @@ -34,13 +35,6 @@ import ( // wrong now as observations, or an error when it could not tell — never "nothing" for "could not // look" (ADR 0227 rule 4). -// awaitingPushBound is how long a machine may have something only its next push makes — an own -// secret of a module assigned since its last push — before that is said (novox/hq issue 275). -var awaitingPushBound = 30 * time.Minute - -// kindAwaitingPush is D1's kind for a machine waiting for a push past awaitingPushBound. -const kindAwaitingPush = "awaiting-push" - // probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator // (mesh-host's `validate`, the package the host runs) takes it. // @@ -80,7 +74,7 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation if err != nil && !unresolvable(err) { return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err) } - var problems, foreseen []string + var problems []string // Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this // resolution as the send judges it: a finding, never a refusal. if err == nil { @@ -89,13 +83,10 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation if err == nil && gensErr == nil { var declared sendable if declared, _, err = composeQuestion(ctx, open, n.Name, compose.Foreseeing, false); err == nil { - foreseen = declared.foreseen // With the order it was last sent, so the validator reads the envelope a machine is sent // — its epoch included (novox/hq to-be 45 §6). var serr error - if declared.Sequence, serr = open.inventory.Sequence(ctx, n.ID); serr == nil { - declared.Epoch, serr = open.inventory.SentEpoch(ctx, n.ID) - } + declared.Sequence, serr = open.inventory.Sequence(ctx, n.ID) if serr != nil { return nil, serr // the store, not the machine: the probe could not run } @@ -121,30 +112,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s", n.Name, len(problems), problems[0]), Said: strings.Join(problems, "; ")}) - case len(foreseen) > 0: - since, err := open.inventory.AwaitingSince(ctx, n.Name) - if err != nil { - return nil, err // the store, not the machine: the probe could not run - } - if waited := time.Since(since); waited > awaitingPushBound { - out = append(out, awaitingPush(n.Name, foreseen, since, waited)) - } } } return out, nil } -// awaitingPush is D1's warning for a machine whose declaration composes only once its next push has -// made what it names, past awaitingPushBound (novox/hq issue 275). -func awaitingPush(node string, foreseen []string, since time.Time, waited time.Duration) conditions.Observation { - made := strings.Join(foreseen, ", ") - return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: kindAwaitingPush, - Kind: kindAwaitingPush, Machine: node, Severity: conditions.Warning, - Summary: fmt.Sprintf("%s has waited %s for a push: its declaration composes once the next push makes %s — "+ - "`push %s` sends it", node, waited.Round(time.Minute), made, node), - Said: fmt.Sprintf("waiting since %s; the next push makes %s", since.UTC().Format(time.RFC3339), made)} -} - // probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its // address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl // takes as final (issue 262). @@ -569,56 +541,53 @@ func holdingNeedsAnAnswer(s catalogue.Seat) bool { return false } -// reportGrace is how long a machine is given, after it was sent a declaration, to apply it and report -// before what the declaration carries is expected to answer (novox/hq issue 275). -var reportGrace = 10 * time.Minute - -// lastSend is what a machine was last sent, as far as the self-check needs it: which modules the send -// carried, when, and whether the machine has reported acting on it. +// lastSend is what a machine runs, as far as the self-check needs it: the modules whose have on it is running (novox/hq ADR +// 0539 §9). It keeps its old name, which said what was sent. type lastSend struct { - // carried is the modules its last send carried; nil when that was not kept (a send from before - // it was, or one by hand), and then every module is taken as carried — as before this existed. - carried map[string]string - // sent is when its current declaration went to it; nil if nothing ever did. - sent *time.Time - // current says its last report names the declaration last sent. - current bool + // running are the modules the node-engine says it runs there. + running map[string]bool } -// settled says whether a module's holder on this machine can be asked to answer now: the machine was -// sent a declaration carrying it, and has reported acting on that declaration or had reportGrace to. -// A machine never sent anything holds nothing the mesh put there. -func (d lastSend) settled(module string, now time.Time) bool { - if d.sent == nil { - return false - } - if d.carried != nil { - if _, in := d.carried[module]; !in { - return false +// settled says whether a module's holder on this machine can be asked to answer now: the node-engine's have of the module is +// running. A module assigned and not applied yet is not there to answer, and a machine that has said nothing holds nothing the +// mesh put there. now is unused: the have says it, with no grace to wait out. +func (d lastSend) settled(module string, _ time.Time) bool { return d.running[module] } + +// runningModules reads from the haves which modules run on which node. Pure. +func runningModules(haves map[string]sdkdelivery.Have) map[string]lastSend { + out := map[string]lastSend{} + for key, h := range haves { + node, module, isNode, err := sdkdelivery.ParseHaveKey(key) + if err != nil || isNode || h.Phase != sdkdelivery.PhaseRunning { + continue } + d := out[node] + if d.running == nil { + d.running = map[string]bool{} + } + d.running[module] = true + out[node] = d } - return d.current || now.Sub(*d.sent) > reportGrace + return out } -// readDeliveries is every machine's last send, by name, from the records a send and a report keep. +// readDeliveries is every machine's running modules, by name, from the node-engines' haves. A process with no bus reads none: +// nothing is then taken as running, never everything. func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]lastSend, error) { - reports, err := inv.LastReports(ctx) + bus, err := aBus() + if err != nil { + return map[string]lastSend{}, nil // empty-on-error: no bus to read the haves from, so no module is taken as running + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) if err != nil { return nil, err } - out := map[string]lastSend{} - for _, r := range reports { - builds, known, err := inv.SentBuilds(ctx, r.Node) - if err != nil { - return nil, err - } - d := lastSend{sent: r.Sent, current: r.Current} - if known { - d.carried = builds - } - out[r.Node] = d + _, haves, err := readWantsAndHaves(ctx, js) + if err != nil { + return nil, err } - return out, nil + return runningModules(haves), nil } // heardMachines is every machine within its heartbeat's bound, as the watchdogs last saw. @@ -1087,21 +1056,17 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, if err != nil { return nil, err } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } shelf, err := inv.Catalogue(ctx) if err != nil { return nil, err } hostVersions := deliveredVersions(shelf[hostModule]) - rolling := rollingModules(plans) + rolling := map[string]bool{} // no plan rolls a module out any more (ADR 0539 §17) nodes, err := inv.Nodes(ctx) if err != nil { return nil, err } - reports, err := inv.LastReports(ctx) + reports, err := lastReports(ctx, inv) if err != nil { return nil, err } @@ -1125,25 +1090,6 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)", n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit))) } - assigned, err := inv.Assigned(ctx, n.Name) - if err != nil { - return nil, err - } - tools := current[broker.RuntimeModule].Commit - if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] { - sent, known, err := inv.SentBuilds(ctx, n.Name) - if err != nil { - return nil, err - } - switch { - case !known: - case !sameCommit(sent[broker.RuntimeModule], tools): - behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s", - short(orNotKnown(sent[broker.RuntimeModule])), short(tools))) - case !applied[n.Name]: - behind = append(behind, "the node tools it was last sent, not yet reported applied") - } - } if len(behind) > 0 { out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind", Machine: n.Name, Severity: conditions.Warning, @@ -1154,26 +1100,6 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, return out, nil } -// rollingModules is every module an open plan is rolling out: those it keeps a record of, and those its -// tiers name. **A release keeps no record per module** — its walk is per machine, its modules only in its -// tier — so reading the records alone, D10 said "no plan is rolling them out" about the node-engine while -// a release walked it, and the gate on that release's first machine waited on what its own send causes -// (novox/hq issue 348). Pure. -func rollingModules(plans []inventory.Plan) map[string]bool { - rolling := map[string]bool{} - for _, p := range plans { - for m := range p.Modules { - rolling[m] = true - } - for _, tier := range p.Tiers { - for _, m := range tier { - rolling[m] = true - } - } - } - return rolling -} - // deliveredVersions are the versions a module's registered build is delivered as: the last element // of every resource path under a `versions/` directory, which registration filled from the artifact's // digest (catalogue `${version}`). The node-engine names itself by that directory. diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index 3015d102..76c30286 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -56,7 +56,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/outward" ) @@ -952,26 +951,10 @@ func setLayerIn(open *stores) setOnWarrant { changed = "nothing changed" } } - // **And sends it** (novox/hq ADR 0344): the layer the operator approved is walked to the machines it reaches - // at once. A walk that cannot be opened is said as the act's failure — the layer is set and nothing sends it. - walked, err := openSettingsWalk(ctx, open, inventory.PlanSettings{Module: p.Module, Node: p.Node, Ask: askID, - Change: changed}) - if err != nil { - return "", fmt.Errorf("%s (ask %s) is set, and the walk that sends it could not be opened: %v — `push %s` sends it", - changed, askID, err, pushTo(p.Node)) - } - return changed + " (ask " + askID + "); " + walked, nil + return changed + " (ask " + askID + ")", nil } } -// pushTo is the push that sends a layer by hand when its walk could not: the machine, or every one behind. -func pushTo(node string) string { - if node == "" { - return "--behind" - } - return node -} - // keyOfLine is what a refusal names of a line: the key of a change's line ("+ key: …"), or the line's first words. func keyOfLine(line string) string { if len(line) > 2 && strings.ContainsRune("+~-=", rune(line[0])) && line[1] == ' ' { diff --git a/cmd/mesh-controller/provider_hold_test.go b/cmd/mesh-controller/provider_hold_test.go deleted file mode 100644 index 5212a9f7..00000000 --- a/cmd/mesh-controller/provider_hold_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one -// unhealthy database provider and three consumers failing their checks that need it — one condition, at the -// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is -// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised -// on its own from the start. -func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - k := conditionsFrom - register(t, open, catalogue.Manifest{Module: "db", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) - consumers := []string{"shop", "wiki", "crm"} - for _, c := range consumers { - register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}}) - } - if _, err := inv.Assign(ctx, "anchor", "db"); err != nil { - t.Fatal(err) - } - machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"} - for _, c := range consumers { - if _, err := inv.Assign(ctx, machineOf[c], c); err != nil { - t.Fatal(err) - } - if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c, - Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { - t.Fatal(err) - } - } - - at := h0 - say := func(machine string, rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy, - Reason: "its command: the database refuses connections", Check: "exec"} - dbUp := dbDown - dbUp.State, dbUp.Reason = link.StateHealthy, "" - failing := func(module string) link.ResourceHealth { - return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} - } - fine := func(module string) link.ResourceHealth { - r := failing(module) - r.State, r.Reason = link.StateHealthy, "" - return r - } - openKeys := func() []conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - - // Two looks of the provider down and its consumers failing, in either order on each machine. - for look := 0; look < 2; look++ { - say("laptop", failing("shop"), failing("wiki")) - say("anchor", dbDown, failing("crm")) - } - raised := openKeys() - if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" { - var keys []string - for _, c := range raised { - keys = append(keys, c.Key) - } - t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys) - } - c := raised[0] - if c.Severity != conditions.Urgent { - t.Errorf("consumers wait on the provider and its condition is %s", c.Severity) - } - said := c.Evidence[0].Said - for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} { - if !strings.Contains(said, w) { - t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said) - } - } - - // Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too. - f, err := gatherGateFacts(ctx, open, "") - if err != nil { - t.Fatal(err) - } - f.open, f.openErr, f.judged = raised, nil, false - for _, m := range consumers { - h, why := moduleHealthWord(m, machineOf[m], h0, f) - if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") { - t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why) - } - } - // And a whole judging past the bound puts nothing back: it waits. - long := h0.Add(-time.Hour) - for _, m := range []string{"anchor", "laptop"} { - f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true} - } - gatherWas := gatherGateFacts - defer func() { gatherGateFacts = gatherWas }() - gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil } - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long} - verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now()) - if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") { - t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last) - } - - // The provider healthy again: a consumer still failing is now its own, at once (its streak stood). - say("anchor", dbUp, fine("crm")) - say("laptop", failing("shop"), fine("wiki")) - var keys []string - for _, c := range openKeys() { - keys = append(keys, c.Key) - } - if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) { - t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys) - } -} - -// A consumer failing while its provider is healthy is raised on its own. -func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - register(t, open, catalogue.Manifest{Module: "db", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) - for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", - Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { - t.Fatal(err) - } - healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"} - shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy, - Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} - for i := 1; i <= 2; i++ { - at := h0.Add(time.Duration(i) * time.Minute) - _ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at) - if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil { - t.Fatal(err) - } - } - list, _ := conditionsFrom.Open(ctx) - if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" { - t.Fatalf("a consumer failing beside a healthy provider raised %v", list) - } -} diff --git a/cmd/mesh-controller/provisioned.go b/cmd/mesh-controller/provisioned.go deleted file mode 100644 index 0c888a17..00000000 --- a/cmd/mesh-controller/provisioned.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "os" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6, -// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the -// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well. - -// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it -// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words. -var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) { - var raw []byte - var said bool - read := func(js nats.JetStreamContext) error { - var err error - raw, said, err = broker.LastProvApplied(js, node, module) - return err - } - var err error - if d := doctorFrom; d != nil && d.js != nil { - err = read(d.js.Context()) - } else { - err = onTheBus(func(conn *nats.Conn) error { - js, err := conn.JetStream(nats.Context(ctx)) - if err != nil { - return err - } - return read(js) - }) - } - if err != nil || !said { - return inventory.ProvReport{}, false, err - } - var r inventory.ProvReport - if err := json.Unmarshal(raw, &r); err != nil { - return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w", - node, module, err) - } - return r, true, nil -} - -// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which -// leaves the machine waiting, then silent, never applied — and is said. -func provLookup(ctx context.Context) inventory.ProvLookup { - return func(node, module string) (inventory.ProvReport, bool) { - r, said, err := provAppliedOn(ctx, node, module) - if err != nil { - fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err) - return inventory.ProvReport{}, false - } - return r, said - } -} - -// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from -// the stream that keeps their reports. -func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) { - all, err := broker.ProvAppliedReports(js) - if err != nil { - return nil, err - } - var out []inventory.ProvReport - for subject, raw := range all { - var r inventory.ProvReport - if err := json.Unmarshal(raw, &r); err != nil { - return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err) - } - if r.Status == inventory.ProvFailed { - out = append(out, r) - } - } - return out, nil -} - -// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue -// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The -// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides -// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send. -func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool, - prov inventory.ProvLookup) (string, bool) { - if len(sent.Provisioned) == 0 { - return r.Outcome, onSend - } - engine := inventory.AppliedReport{Outcome: r.Outcome} - if r.At != nil { - engine.At = *r.At - } - got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov) - return got.Outcome, ok -} diff --git a/cmd/mesh-controller/provisioned_test.go b/cmd/mesh-controller/provisioned_test.go index a9756c35..71ecac88 100644 --- a/cmd/mesh-controller/provisioned_test.go +++ b/cmd/mesh-controller/provisioned_test.go @@ -55,46 +55,6 @@ func TestTheProvisionersAccountIsTheClientsProcessUser(t *testing.T) { } } -// The first-machine gate counts a handed module healthy only once its node's prov.applied for the new build has -// come (ADR 0349 §15, kept by ADR 0362 for a walk finishing at its gate); a failure there fails it. -func TestTheGateWaitsForTheProvisionersReport(t *testing.T) { - now := time.Date(2026, 10, 14, 23, 0, 0, 0, time.UTC) - since := now.Add(-time.Minute) - reported := now.Add(-50 * time.Second) - digest := "sha256:" + strings.Repeat("d", 64) - facts := func(said map[string]inventory.ProvReport) gateFacts { - return gateFacts{now: now, - reports: map[string]inventory.Reported{"g14": {Node: "g14", Outcome: inventory.OutcomeApplied, - At: &reported, Current: true}}, - sent: map[string]inventory.SentDeclaration{"g14": {Digest: "sent", At: since, - Provisioned: map[string]string{"dunst": digest}}}, - provisioned: func(node, module string) (inventory.ProvReport, bool) { - r, ok := said[node+"."+module] - return r, ok - }} - } - m := catalogue.Manifest{Module: "dunst"} - if h, why := judgeHealth("dunst", "", m, "g14", since, facts(nil)); h != healthNotYet || - !strings.Contains(why, "has not reported dunst's files placed") { - t.Fatalf("judged without the provisioner's report: %v %q", h, why) - } - failed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvFailed, Templates: digest, - Reason: "a value is not there", At: now.Add(-10 * time.Second)}} - if h, why := judgeHealth("dunst", "", m, "g14", since, facts(failed)); h != healthBroken || - !strings.Contains(why, "a value is not there") { - t.Fatalf("a failed placing is not the build's failure there: %v %q", h, why) - } - placed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvApplied, Templates: digest, - At: now.Add(-10 * time.Second)}} - if h, why := judgeHealth("dunst", "", m, "g14", since, facts(placed)); strings.Contains(why, "provisioner") { - t.Fatalf("placed, the module is still held on the provisioner: %v %q", h, why) - } - // A module the send did not hand is judged as before. - if h, why := judgeHealth("picom", "", catalogue.Manifest{Module: "picom"}, "g14", since, facts(nil)); strings.Contains(why, "provisioner") { - t.Fatalf("a module not handed waits on the provisioner: %v %q", h, why) - } -} - // A machine whose provisioner client failed to place a module's files has a failed apply, said as apply-failing // once the failure outlasts an apply interval, with the client's reason (ADR 0362 §3). func TestAFailedPlacingRaisesApplyFailing(t *testing.T) { diff --git a/cmd/mesh-controller/provisioning_failed.go b/cmd/mesh-controller/provisioning_failed.go new file mode 100644 index 00000000..a3617a04 --- /dev/null +++ b/cmd/mesh-controller/provisioning_failed.go @@ -0,0 +1,31 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from +// the stream that keeps their reports. +func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) { + all, err := broker.ProvAppliedReports(js) + if err != nil { + return nil, err + } + var out []inventory.ProvReport + for subject, raw := range all { + var r inventory.ProvReport + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err) + } + if r.Status == inventory.ProvFailed { + out = append(out, r) + } + } + return out, nil +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 1e8efc6f..651765ec 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -4,41 +4,21 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" - "flag" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/conditions" - "io" "log" "os" - "slices" - "sort" "strings" - "time" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/compose" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) -// reportUnhostable says which of a node's assigned modules the machine cannot run, once per push. -// -// A module whose declared capability has no detector on the machine is on the wrong machine. It is -// kept out of what the node is sent — the healthy modules beside it still converge — and named here -// so it is neither silently dropped nor a reason the whole node fails to push. -func reportUnhostable(node string, plan catalogue.Resolution) { - for _, u := range plan.Unhostable { - for _, c := range u.Missing { - fmt.Printf("%s not applied — %s\n", node, catalogue.WrongMachine(u.Module, c, node)) - } - } -} - // sending it, and holding the link that carries it. // // Split out of main.go, which had reached 2,769 lines because appending was always the @@ -189,31 +169,21 @@ func serve(ctx context.Context) (err error) { // by a replaced controller resumes from the first look, which this asks for. No timer looks every so often. // **Its own grants on the bus before it writes** (novox/hq issue 1800): a bucket this controller newly writes is // granted before the first write, not after its gate. - startWithOwnGrants(ctx, ownGrantsOf(open), func() { startRolloutEvents(ctx, open, server.JetStream()) }) - // And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes, - // which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it; - // nothing polls (issue 973). - if err := startMergeWindow(ctx, open, server.JetStream()); err != nil { - return fmt.Errorf("the merge window could not be put on the bus: %w", err) - } + startWithOwnGrants(ctx, ownGrantsOf(open), func() {}) // And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module // is registered, ended with why when its build will not register it, raised and cleared as conditions. // Never by a read. go settlingPending(ctx, open) // The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0). - go forgettingOldDurations(ctx, inv) // And what the catalogue decided a build meant. The builder's own result is already handled // above; this is the other half — the control plane is the only one of the three that knows // which machines run the thing, so it is the one that acts (novox/hq ADR 0072). if err := server.Follows(following{open: open}); err != nil { return err } - // And the merges it did not hear while it was away, caught up once it serves (novox/hq ADR 0457, issue 1336). - go catchUpAtStart(ctx, open) // And the merges mesh-delivery made, kept a week with the ask each was made on: what registers a module that // moved between repositories on the operator's Approve (novox/hq ADR 0420 §7, issue 1025). - if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: open.inventory.Catalogued, - kept: following{open: open}.resumeFirstBuilds}); err != nil { + if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: open.inventory.Catalogued}); err != nil { return err } // No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable @@ -255,8 +225,11 @@ func serve(ctx context.Context) (err error) { if !isNATS { return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it") } - // The hand-act log is counted for `status` on this connection rather than a new one a minute. + // The hand-act log is counted for `status` on this connection rather than a new one a minute; no longer serving, the + // connection is closed, and a verb run after in this process dials its own (a served test's leftover failed the + // next test's `conditions` with "connection closed", issue 1945). handActConn = bus.Conn + defer func() { handActConn = nil }() // And everything else this controller does on the bus for a moment (novox/hq issue 327). servingBus.Store(server.JetStream()) // No longer serving: nothing is lent, and dead-letters says it is not read here (novox/hq issue 330). @@ -267,7 +240,6 @@ func serve(ctx context.Context) (err error) { // (novox/hq to-be 45 §9). checkEvents = bus // And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239). - inventory.PlanSaved = servingPlanSaved(ctx, bus, open) if err := server.Checks(following{open: open}); err != nil { return err } @@ -287,6 +259,8 @@ func serve(ctx context.Context) (err error) { // And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by // the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build. watchNewModules(ctx, open) + // And a value given by hand replaced when its module's have first runs healthy (novox/hq ADR 0228, ADR 0539 §9). + watchHavesForGiven(ctx, open) // And says what it keeps, and keeps what every module says it keeps (novox/hq ADR 0495, issue 1790): a changed word // sends the machine holding the bus its new user list. if err := announceOwnStates(ctx, server.JetStream()); err != nil { @@ -339,854 +313,12 @@ func serve(ctx context.Context) (err error) { return server.Serve(ctx) } -// declare sends one node a declaration, signed. -// -// Signed here rather than trusted from the broker: a node connects to the broker and takes -// instruction from the control plane behind it, and those are two identities. If a node believed -// whatever arrived on its queue, a compromised broker could forge declarations — and since the -// host applies whatever the link delivers, that is the whole machine (novox/hq ADR 0004). -func declare(ctx context.Context, args []string) error { - if len(args) != 2 { - return errors.New("declare ") - } - node, path := args[0], args[1] - - raw, err := os.ReadFile(path) - if err != nil { - return err - } - - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // The node has to exist before it can be told anything. Publishing to a queue nobody consumes - // would sit there looking like success. - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - if _, err := inv.NodeByName(ctx, node); err != nil { - return err - } - - // **With the inventory, so the bus is raised** (novox/hq ADR 0134, design 30). A module's - // declaration and how it hears what it consumes move together: its consumer is derived from the - // same records this declaration is composed from. Raised only when the control plane started - // serving, a module that gained a `consumes` was sent a declaration it could act on and a - // consumer that never delivered the event — and nothing anywhere said the two disagreed - // (found on review, 2026-09-28). Everything the raise does is idempotent. - server, err := connectLink(ctx, inv, nil, nil) - if err != nil { - return err - } - defer server.Close() - - if err := link.Declare(ctx, server.Bus(), ident, node, raw, 15*time.Second); err != nil { - return err - } - // Written down like every other send (novox/hq issue 204): a declaration a person sent by hand - // is still what the machine was last told, and status must not read it as current for the one - // the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259): - // the mesh did not compose it, so a push that does not name this machine treats it as held. - // The epoch it carried, if a person wrote one in, is what the machine heard. - var carried struct { - Epoch uint64 `json:"epoch"` - Sequence int64 `json:"sequence"` - Generation int64 `json:"generation"` - } - _ = json.Unmarshal(raw, &carried) - // And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it - // named are not known, since the mesh did not compose it. - if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence, - epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation, - sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil { - return err - } - fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) - return nil -} - // OverlayCIDRVar is the range the mesh allocates node addresses from. const OverlayCIDRVar = "MESH_OVERLAY_CIDR" -// pushCommand sends nodes everything they should be: their place on the network, and what their -// assignments resolve to. -// -// One declaration, not two. A node holding its network and not its modules, or the reverse, is -// half-configured for as long as that lasts — and the two are computed from the same picture of -// the mesh, so sending them apart would let them disagree. -func pushCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("push", flag.ContinueOnError) - // Only the machines that need it. - // - // **A command rather than a timer, to begin with.** Something that re-pushes on a schedule is - // a scheduler over this, and building the scheduler first would mean two paths to one act - // with nothing to compare them against. A person can run this; so can cron; so can whatever - // eventually watches. - behind := set.Bool("behind", false, - "only machines whose last declaration was refused or partly failed") - // For a named node, wait until it reports applying exactly what it was sent, so `push ` - // means "this node is now what it was told" — a command right after does not race the apply - // (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget. - wait := set.Duration("wait", 0, - "for a named node, how long to wait for it to report applying what it was sent (0: do not wait)") - // A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat, - // recorded when given at a shell — see handacts.go for why a shell is not refused. - why := addHandActFlags(set) - // Every machine at once is said, not defaulted to (novox/hq ADR 0217): a bare `push` sent the - // whole mesh on 2026-10-05 when its usage was wanted. - all := set.Bool("all", false, "every machine") - // A running module's data moving is sent only when said, per module (novox/hq ADR 0217). - move := set.String("move", "", "modules whose data may move with this push, comma-separated") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - args = positionals - if len(args) > 1 { - return errors.New("push | push --behind | push --all — one machine, the ones behind, or all of them") - } - if len(args) == 0 && !*behind && !*all { - return errors.New("push names a machine, or says --behind (the ones not running what they " + - "should) or --all (every machine) — a push to the whole mesh is not the default (novox/hq ADR 0217)") - } - if *all && (*behind || len(args) == 1) { - return errors.New("push --all is every machine; it takes no machine and no --behind") - } - movable := map[string]bool{} - for _, m := range splitModules(*move) { - movable[m] = true - } - var heldBack []string - if len(args) == 1 && *behind { - // Naming a machine and asking for the ones that need it are two different requests, and - // guessing which was meant would sometimes push to a machine somebody did not name. - return errors.New("push or push --behind, not both: one names a machine and the " + - "other asks which machines need one") - } - recorded := append([]string(nil), args...) - if *behind { - recorded = append(recorded, "--behind") - } - if *all { - recorded = append(recorded, "--all") - } - if *move != "" { - recorded = append(recorded, "--move", *move) - } - why.record(ctx, "push", recorded) - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // Every node, not only the ones on the private network. A machine that was never given the - // network module still takes modules, and iterating the network here is what used to make - // "on the network" and "managed" the same thing. - nodes, err := inv.Nodes(ctx) - if err != nil { - return err - } - - // Which machines are not in the state they were sent, when that is what was asked for. - var needsOne map[string]inventory.Doing - if *behind { - wrong, err := inv.NotDoingWhatTheyWereTold(ctx) - if err != nil { - return err - } - needsOne = map[string]inventory.Doing{} - for _, d := range wrong { - needsOne[d.Node] = d - } - // **And every machine not running what the mesh would send it.** "Behind" used to mean - // only "failed or refused", so a machine that applied cleanly and whose declaration has - // since changed was not behind — and novox/hq ADR 0010's question, *did my change go - // out?*, was answerable only for the machines that broke. - would, err := wouldSend(ctx, open, nodes) - if err != nil { - return err - } - waiting, err := inv.Waiting(ctx, would) - if err != nil { - return err - } - for _, m := range waiting { - if _, already := needsOne[m.Node]; already { - continue - } - needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"} - } - if len(needsOne) == 0 { - // Said rather than doing nothing quietly. "Nothing needed one" and "this did not run" - // must never look the same. - fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent") - return nil - } - } - // The private network every declaration of the push is composed with: not computable refuses the push up front. - if _, err := generatorsOf(ctx, open); err != nil { - return err - } - - server, err := connectLink(ctx, nil, nil, nil) - if err != nil { - return err - } - defer server.Close() - - // Which machines this push is about, before any of them is worked out. - var asked []string - for _, n := range nodes { - if len(args) == 1 && n.Name != args[0] { - continue - } - if *behind { - doing, needs := needsOne[n.Name] - if !needs { - continue - } - // A machine that has been failing the same way for a long time is not going to stop - // because it was asked again. Said, and pushed to anyway — refusing would leave no - // way to retry after fixing the cause, and this is a command somebody ran. - // - // Only for machines that reported something. One that is merely waiting has no report - // to be old, and saying it had been failing since the zero time would be a sentence - // about nothing. - if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour { - fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+ - "unlikely to be timing\n", - n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04")) - } - } - asked = append(asked, n.Name) - } - // **A push that named no machine says so, first.** Through the console a machine the caller - // meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as - // `push --behind`, and every machine behind was pushed by someone who thought they had pushed one. - // Its whole-mesh reach is the first line of the answer, with the machines it is about to send. - if len(args) == 0 { - which := "every machine" - if *behind { - which = "every machine that is behind" - } - fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n", - which, len(asked), strings.Join(asked, ", ")) - } - - // **A whole-mesh push sends no build a gate has not seen** (novox/hq ADR 0236): a machine where one - // waits is left, named, for the release plan — `push ` still sends one machine by a person's word. - if len(args) == 0 { - f, err := readMoveFacts(ctx, inv) - if err != nil { - return err - } - var kept []string - for _, n := range asked { - moves, err := machineMoves(ctx, open, f, n, false) - if err != nil { - return err - } - if len(moves) > 0 { - fmt.Printf(" %s is left: %d build(s) wait there for a gate, which a release plan sends one machine "+ - "at a time (`upgrade backlog` lists them; `push %s` sends it by name)\n", n, len(moves), n) - continue - } - kept = append(kept, n) - } - asked = kept - } - // **The bus is replaced only as a planned step** (novox/hq ADR 0236, to-be 45 §8): a machine whose bus - // would move is not sent by a push — named, it is refused; otherwise it is left and said. - busKept, err := busHeld(ctx, inv, asked) - if err != nil { - return err - } - if len(busKept) > 0 { - if len(args) == 1 { - return fmt.Errorf("%s. Nothing was sent", busKept[args[0]]) - } - var kept []string - for _, n := range asked { - if why, h := busKept[n]; h { - fmt.Printf(" %s is left: %s\n", n, why) - continue - } - kept = append(kept, n) - } - asked = kept - } - - // **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's - // user list, and a module's new grants are refused by the bus until that list says them. Among - // the machines asked it goes first; not among them and behind, it is added — a named push whose - // module gained a state would otherwise send the code and leave the right to use it for the - // cascade below, after. - holder, holderBehind, err := brokerBehind(ctx, open, asked) - if err != nil { - return err - } - // **Not when its own modules are held back** (novox/hq issue 259, ADR 0221): added rather than - // named, it is sent its whole declaration, and a build its policy records or a plan has not sent - // it yet would go with the user list. Named and left, with what that costs. - saidHeld := map[string]bool{} - if holderBehind && len(args) == 1 { - held, err := heldMachines(ctx, open, []string{holder}) - if err != nil { - return err - } - if why, isHeld := held[holder]; isHeld { - sayHeld(os.Stdout, holder, why) - fmt.Printf("%s holds the bus, and the user list it would carry has changed: until it is "+ - "sent, the bus may refuse what this push's machines were newly granted\n", holder) - holderBehind = false - saidHeld[holder] = true - } - } - asked = brokerFirst(asked, holder, holderBehind) - - // Held from composing to sending, so a converge on one of them cannot send between the two - // and be overtaken by what was composed before it (novox/hq ADR 0100). - held, release, err := holdNodes(ctx, open, asked) - if err != nil { - return err - } - // Each machine's unmet seat dependencies (novox/hq ADR 0207), said after the sends: in full - // for a machine named, as a count for each of many — the full list is `status`'s. - unheld := map[string][]catalogue.Unheld{} - sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) { - // What this machine's send reads is read afresh: the send before it made and recorded things. - held := freshInput(held) - plan, _, err := resolveNode(held, open, node) - if err != nil { - return sendable{}, err - } - // A module assigned here that this machine cannot host is said and left out, not fatal: the - // healthy modules beside it are still resolved and sent. Reported so it is not silently - // dropped — the remedy is to move it, and until then the rest of the node converges. - reportUnhostable(node, plan) - reportKept(held, plan) - unheld[node] = plan.Unheld - // The private network is in here with everything else. It used to be composed separately - // and prepended, which meant every machine with an address was on it and no machine could - // be kept off. It is a module now, so it arrives the way a module does. - declared, err := composeSend(held, open, node) - if err == nil { - reportLeftOut(node, declared) - } - return declared, err - }) - - defer release() - // A machine whose declaration would move a running module's data is held, and only it (novox/hq - // ADR 0217): said, with the command that sends it, and left out of everything below. - toSend, err := holdingBack(ctx, inv, sending, movable, &heldBack) - if err != nil { - return err - } - // **What it recreates, said before it is sent** (novox/hq ADR 0245): a person's push moves every - // module whose build changed, and recreates what changed in it — a gated send said so, a push did not. - sayWhatAPushRecreates(ctx, open, toSend, os.Stdout) - // Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push - // is the one most operators run, and on 2026-10-01 it was the one path that issued none. - bus := overTheBus{open: open, server: server, signer: ident} - sentDigest, err := deliver(ctx, bus, holder, toSend) - if err != nil { - return err - } - release() - told := make([]string, 0, len(toSend)) - for _, r := range toSend { - told = append(told, r.node) - } - fmt.Printf("\n%d node(s) told: %s\n", len(toSend), strings.Join(told, ", ")) - reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld) - - // **A named push leaves the mesh consistent, not just the machine it named** (novox/hq - // issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's - // grant list is a pure read of secrets already issued — so after the named node is current, - // other machines can be behind *as a consequence*: their declaration now differs from what - // they were last sent. Those are flushed too, by name, in the push's own output. - // - // Compared against what each machine was last SENT, not against a before/after of this push: - // the mint usually happened at `assign` or `module issue`, before this command ran, so the - // only durable signal is "what it should be" versus "what it last received". Bounded: a - // flushed send may itself mint, so this converges over a few rounds. - // - // **Except a machine a policy or a plan holds back** (novox/hq issue 259, ADR 0221): one whose - // modules would move to a build their upgrade policy records rather than rolls out, or that an - // open plan has not sent it yet. It is named, with why, and left for a push that names it. - if len(args) == 1 { - handled := map[string]bool{args[0]: true} - for n := range saidHeld { - handled[n] = true - } - // The cascade is a push too, and a machine in it whose data would move is held the same way - // (novox/hq ADR 0217). - keep := keeping(func(held context.Context, sending []readyNode) ([]readyNode, error) { - return holdingBack(held, inv, sending, movable, &heldBack) - }) - refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open), bus, holder, keep, os.Stdout) - refusals = append(refusals, refused...) - if err != nil { - return err - } - } - - // A named node is a request to make THAT node current now, so it waits for the node to say it - // applied exactly this. A whole-mesh or --behind push does not wait: it is a sweep, and blocking - // on the slowest machine would hold back the report on all the others. - if *wait > 0 && len(args) == 1 { - if err := waitForApplied(ctx, inv, args[0], sentDigest[args[0]], *wait); err != nil { - return err - } - } - if len(heldBack) > 0 { - sort.Strings(heldBack) - return fmt.Errorf("held %s: a running module's data would move — see above; nothing was sent "+ - "there (novox/hq ADR 0217)", strings.Join(heldBack, ", ")) - } - return couldNotBeResolved(refusals, len(sending)) -} - -// waitForApplied blocks until the node reports it applied exactly the declaration just sent, or the -// wait runs out. A report of failure or refusal for that same declaration ends the wait at once — -// there is nothing to wait for, and the reason is the node's own. -func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest string, wait time.Duration) error { - if digest == "" { - return nil // nothing was sent to this node - } - deadline := time.Now().Add(wait) - for { - doing, said, err := inv.DoingOf(ctx, node) - if err != nil { - return err - } - if said && doing.Declared == digest { - switch doing.Outcome { - case inventory.OutcomeApplied: - fmt.Printf("%s applied it\n", node) - return nil - case inventory.OutcomeFailed: - // Each failed resource by id with the machine's own words, so the answer says what to fix - // without a second look (novox/hq issue 629). - var each []string - for _, f := range doing.Failed { - each = append(each, f.ID+": "+f.Error) - } - return fmt.Errorf("%s applied what it was sent but %d resource(s) failed: %s", node, len(doing.Failed), - strings.Join(each, "; ")) - case inventory.OutcomeRefused: - return fmt.Errorf("%s refused what it was sent: %s", node, doing.Refused) - } - } - if time.Now().After(deadline) { - return fmt.Errorf("%s did not report applying what it was sent within %s "+ - "(it may still be converging; check `status`)", node, wait) - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(500 * time.Millisecond): - } - } -} - -// readyNode is one machine and the declaration it would be sent. -type readyNode struct { - node string - declared sendable -} - -// composeEach works out what each named machine should be, and never lets one machine's answer -// decide another's. -// -// **A machine whose set cannot be worked out is that machine's problem** (novox/hq ADR 0066). A -// whole-mesh push used to refuse outright when any one node failed to resolve, so a single -// unanswerable requirement on a single machine — one module requiring a provision nobody had -// assigned a provider for — left every other machine in the mesh unconverged, including machines -// with no relation to it at all. Nothing was sent anywhere, and the machines that could not be sent -// were the ones with nothing wrong with them. -// -// It is the same rule a92c11b established one level down, where an un-hostable module stopped -// taking down the healthy modules beside it, applied one level up: **the blast radius of a fault is -// the thing that has it.** What could not be worked out is named and returned, so a push still ends -// with a non-zero outcome and nobody mistakes a partial convergence for a whole one. -// -// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential -// across two machines that must agree — and dropped here, where it never did. -func composeEach(names []string, allot func(node string) (order, error), - compose func(node string) (sendable, error)) ([]readyNode, []string) { - - var sending []readyNode - var refusals []string - for _, name := range names { - // **Numbered before it is composed, not before it is sent** (novox/hq issue 204). The - // number says where this declaration stands against every other the mesh composed for the - // machine, and the host refuses one lower than the last it applied. Taken at send time, as - // it was, a declaration composed a minute ago — before an assignment changed — went out with - // a number higher than one composed after the change and sent before it, and the machine - // took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on - // two machines was undone two seconds later by exactly that. Taken here, before the first - // read, what was composed earlier is numbered lower whatever order the sends happen in. - numbered, err := allot(name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - declared, err := compose(name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - numbered.stamp(&declared) - if len(declared.Resources) == 0 { - // Sent, not skipped (novox/hq issue 127). A node whose declaration composes to - // nothing may have HELD something before — the broker opening a placement gave it, - // say — and skipping the empty declaration leaves that last resource in force - // forever, re-applied by the node's own heartbeat, with no way for the mesh to say - // it is gone. An empty declaration is the correction: the host drops what the mesh - // owned and keeps what it found (the adoption envelope still rides along). A node - // that never held anything applies it as the no-op it is. - fmt.Printf("%s owns nothing now — sent so it drops what it last held\n", name) - } - sending = append(sending, readyNode{name, declared}) - } - return sending, refusals -} - -// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold -// back on every way out — a body that cannot be marshalled and a send that fails included -// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are -// still sent. Their memberships go before their declarations, as every send's do (issue 249). -func sendRound(ctx context.Context, open *stores, names []string, - compose func(held context.Context, node string) (sendable, error), - d delivery, holder string, keep keeping) ([]string, error) { - held, release, err := holdNodes(ctx, open, names) - if err != nil { - return nil, err - } - defer release() - sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) { - return compose(held, node) - }) - if keep != nil { - if sending, err = keep(held, sending); err != nil { - return refused, err - } - } - if _, err := deliver(held, d, holder, sending); err != nil { - return refused, err - } - return refused, nil -} - -// keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil -// keeps them all. A push holds back the machines whose running modules' data would move (holdingBack). -type keeping func(held context.Context, sending []readyNode) ([]readyNode, error) - -// holdingBack leaves out each machine whose declaration would move a running module's data, says so, -// and names it among those held back (novox/hq ADR 0217); the rest go on to be delivered, grants first -// (issue 249). A declaration's body is the same bytes however often it is read. -func holdingBack(ctx context.Context, inv *inventory.Inventory, sending []readyNode, movable map[string]bool, - heldBack *[]string) ([]readyNode, error) { - var out []readyNode - for _, s := range sending { - body, err := s.declared.Body() - if err != nil { - return nil, err - } - moves, err := heldMoves(ctx, inv, s.node, body, movable) - if err != nil { - return nil, err - } - if len(moves) > 0 { - sayDataHeld(os.Stdout, s.node, moves) - *heldBack = append(*heldBack, s.node) - continue - } - out = append(out, s) - } - return out, nil -} - -// delivery is the two acts of sending machines what they should be, apart, so the order between -// them is one function's and can be read and tested there (novox/hq issue 249). -type delivery interface { - // grant issues what the machines' modules may do — each module's state raised and its - // membership issued — for every machine about to be sent. - grant(ctx context.Context, sending []readyNode) error - // declare sends one machine its declaration and records it sent, answering the digest. - declare(ctx context.Context, s readyNode, body []byte) (string, error) -} - -// errGrants marks a send that stopped because what the machines' modules may do could not be issued -// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an -// announcement that hits it is held and asked again. -var errGrants = errors.New("what the machines' modules may do on the bus could not be issued, and code " + - "sent before its grants is refused there") - -// grantsRefused is a grant that failed for some machines and not others: their memberships could not -// be issued, by machine, and only those machines are held back. -type grantsRefused struct{ nodes map[string]error } - -func (g *grantsRefused) Error() string { - names := make([]string, 0, len(g.nodes)) - for n := range g.nodes { - names = append(names, n) - } - sort.Strings(names) - return fmt.Sprintf("the memberships of %s could not be issued; the first: %v", - strings.Join(names, ", "), g.nodes[names[0]]) -} - -// deliver sends the machines their declarations: **the machine holding the bus, then the grants, -// then the rest** (novox/hq issue 249). -// -// A merge gave a module a new state; its bundle reached every machine within a minute, and the -// machines' permissions on the bus did not include the state until somebody pushed by hand: the code -// arrived before the right to use it. A module that read its new state on start failed its start; the -// one that was there retried for two minutes. The memberships were issued after the declarations — -// "because the runtime it is for arrives with it" — and a membership is retained last-per-subject on -// the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A -// runtime still on the old code merely holds a grant it does not use yet. -// -// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the -// controller's own right to publish memberships and raise buckets is in that list (the precedent of -// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very -// declaration that allows it — a lock only a hand on the broker could open. A runtime already running -// on that machine follows a membership issued after its declaration, as it always has. -// -// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to -// be said and passed over — "the machines keep what they derive until the next push" — which reported -// a rollout done that had delivered code its machines could not run. A membership that failed holds -// back its own machine; a failure that names no machine (the buckets) holds back every machine but the -// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is -// tried again. -// -// The grants are issued, not waited on: a membership is a retained message the runtime reads when it -// comes, and the bus answers its publication; nothing here waits for a runtime to have read one. -func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) { - digests := map[string]string{} - if len(sending) == 0 { - return digests, nil - } - send := func(s readyNode) error { - // The number is inside the signed bytes, so a replayed older declaration cannot borrow a - // newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204). - body, err := s.declared.Body() - if err != nil { - return err - } - digest, err := d.declare(ctx, s, body) - if errors.Is(err, errFilesNotPut) && s.declared.fileReferences { - // **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the - // bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the - // grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no - // references, and says so; the next one tries the bucket again. - fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err) - s.declared.fileReferences = false - if body, err = s.declared.Body(); err != nil { - return err - } - digest, err = d.declare(ctx, s, body) - } - if err != nil { - return err - } - digests[s.node] = digest - return nil - } - var rest []readyNode - for _, s := range sending { - if holder != "" && s.node == holder { - if err := send(s); err != nil { - return digests, err - } - continue - } - rest = append(rest, s) - } - held := map[string]error{} - if err := d.grant(ctx, sending); err != nil { - var some *grantsRefused - if !errors.As(err, &some) { - var names []string - for _, s := range rest { - names = append(names, s.node) - } - if len(names) == 0 { - return digests, fmt.Errorf("%w: %w", errGrants, err) - } - return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err) - } - held = some.nodes - } - var notSent []string - for _, s := range rest { - if _, refused := held[s.node]; refused { - notSent = append(notSent, s.node) - continue - } - if err := send(s); err != nil { - return digests, err - } - } - if len(notSent) > 0 { - return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "), - &grantsRefused{nodes: held}) - } - if len(held) > 0 { - // Only the holder's own memberships failed, and it was sent before them. - return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held}) - } - return digests, nil -} - -// overTheBus is delivery as the mesh does it: memberships on the bus, declarations signed. -type overTheBus struct { - open *stores - server *link.Server - signer link.Signer - // indent is put before each "sent" line, for the callers whose output is nested. - indent string -} - -// putFiles puts the contents s names by digest in the bucket. -func (b overTheBus) putFiles(ctx context.Context, s readyNode) error { - objects, err := s.declared.objects() - if err != nil { - return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err) - } - if len(objects) == 0 { - return nil - } - bus, ok := b.server.Bus().(link.OverNATS) - if !ok { - return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node) - } - putting, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - store, err := openBucket(putting, bus.Conn, s.node) - if err == nil { - err = putObjects(putting, store, objects) - } - notePut(ctx, s.node, err) - if err != nil { - return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err) - } - return nil -} - // errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket. var errFilesNotPut = errors.New("the files it names by digest are not in the bucket") -func (b overTheBus) grant(ctx context.Context, sending []readyNode) error { - // The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised - // when they cannot be, and never what holds the send back (assertOnSend). - if bus, ok := b.server.Bus().(link.OverNATS); ok { - js := broker.OnConn(bus.Conn) - js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) } - _ = assertOnSend(ctx, b.open.inventory, js, b.indent) - } - return issueMemberships(ctx, b.open, b.server, sending) -} - -func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) { - // The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node - // that applies it finds every one. A node reading no references has none. - // **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them. - unlock := lockFilesOf(s.node) - err := b.putFiles(ctx, s) - if err != nil { - unlock() - return "", err - } - declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second) - unlock() - if declared != nil { - return "", declared - } - // After it is away, not before. A digest recorded for something that failed to send would make - // the machine look current for a declaration it never received. - digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch, - sentRecordOf(ctx, s.declared)) - if err != nil { - return "", err - } - // A send that carried the mesh's record of the controller and the node-engine — a person's push included — - // ends the condition of one kept over it (novox/hq issue 201). - settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds) - if len(s.declared.Bindings) > 0 { - // And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232): - // what the next resolution keeps it at. On the same outliving context as the send's record. - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings) - cancel() - if err != nil { - return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+ - "data could not be recorded: %w", s.node, err) - } - } - if s.declared.BusUsers != "" { - // And the user list it carried, so the next send reads whether it must go first from the - // list alone (novox/hq issue 249). On the same outliving context as the send's record. - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers))) - cancel() - if err != nil { - return "", err - } - // The list sent issued the credential of every module it carried, at the fingerprint each is registered at: - // the grants the assigner's sixth rule reads (novox/hq ADR 0528 clause 2, issue 1394). - if bus, ok := b.server.Bus().(link.OverNATS); ok { - if js, err := jetstream.New(bus.Conn); err == nil { - writing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - grantsIssued(writing, b.open.inventory, js, b.indent) - cancel() - } - } - } - fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources)) - return digest, nil -} - -// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine -// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a -// module's new permissions are refused by the bus until that list says them. Among the machines it -// is moved to the front; not among them, it is added only when it is behind. -func brokerFirst(names []string, holder string, behind bool) []string { - if holder == "" { - return names - } - present := false - rest := make([]string, 0, len(names)) - for _, n := range names { - if n == holder { - present = true - continue - } - rest = append(rest, n) - } - if !present && !behind { - return names - } - return append([]string{holder}, rest...) -} - // brokerBehind is the machine holding the bus — the one whose declaration carries the user list — // and, when it is not among the machines named, whether the user list it would be sent now differs // from the one it was last sent (novox/hq issue 249). @@ -1225,291 +357,53 @@ func brokerListState(ctx context.Context, open *stores, names []string) (busList return busListState{Machine: h.Node}, nil } } - // **What cannot be worked out is behind** (the review of hq issue 353). The list it would be sent cannot be - // composed, so whether it carries the grants a send relies on is not known; it is taken as behind, so a push - // sends that machine first and says why it cannot (`plan` says it too), and a rollout that needs the list - // refuses rather than sending code whose grants the bus may not hold. It said "not behind" until then: a - // send went ahead as if the bus held a list nobody could compose. - list, composeErr := "", error(nil) - plan, _, err := resolveNode(ctx, open, h.Node) + // **What cannot be worked out is behind** (the review of hq issue 353): the composition carrying the user list is not + // written, so whether the bus's machine runs it is not known; taken as behind, never "not behind". + // The user list is part of the bus module's composition on the machine holding it (ADR 0539 §15), so the machine is + // behind while its have does not name the digest of the composition written now. + fps, err := currentFingerprints(ctx, inv) if err != nil { - composeErr = err - } else if list, _, err = busUserList(ctx, inv, plan.Modules); err != nil { - composeErr = err + return busListState{Machine: h.Node, Behind: true, Uncomposed: err}, nil } - sent, err := inv.SentBusUsers(ctx, h.Node) + bus, err := aBus() if err != nil { return busListState{}, err } - return busListState{Machine: h.Node, Behind: listBehind(list, composeErr, sent), Uncomposed: composeErr}, nil + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return busListState{}, err + } + digest, digestErr := composedDigestOf(ctx, js, h.Node, h.Module, fps[h.Module]) + have, had, err := haveOn(ctx, js, h.Node, h.Module) + if err != nil { + return busListState{}, err + } + return busListState{Machine: h.Node, Behind: listBehind(digest, digestErr, have, had), Uncomposed: digestErr}, nil } -// listBehind is whether the bus's machine is behind: the list composed now is not the one last sent, or it -// could not be composed at all — not known is behind, never "not behind". -func listBehind(list string, composeErr error, sentDigest string) bool { - return composeErr != nil || userListBehind(list, sentDigest) +// composedDigestOf is the digest of the composition the controller wrote for a module on a node at a fingerprint. +func composedDigestOf(ctx context.Context, js jetstream.JetStream, node, module, fp string) (string, error) { + digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket) + if err != nil { + return "", err + } + key, err := sdkdelivery.ComposedKey(node, module, fp) + if err != nil { + return "", err + } + e, err := digests.Get(ctx, key) + if err != nil { + return "", fmt.Errorf("the composition %s is not written: %w", key, err) + } + v, err := sdkdelivery.DecodeComposedDigest(e.Value()) + return v.Digest, err } -// userListBehind is whether the user list composed now is not the one last sent, by its digest. An -// empty list composed is never behind: there is nothing for it to carry. -func userListBehind(now, sentDigest string) bool { - return now != "" && digestOf([]byte(now)) != sentDigest -} - -// couldNotBeResolved is what a push ends with when some machines could not be worked out. -// -// **After the rest have been sent, never instead of sending them.** It is still an error, because -// the mesh is not in the state somebody asked for and a command that exits cleanly having skipped a -// machine is a command that lies. What it must not do is decide anything about the machines beside -// it, which is why it says how many were sent. -func couldNotBeResolved(refusals []string, sent int) error { - if len(refusals) == 0 { - return nil - } - return fmt.Errorf( - "%d node(s) could not be resolved and were not sent. %d other node(s) were:\n\n%s", - len(refusals), sent, strings.Join(refusals, "\n\n")) -} - -// sendTo resolves and sends to exactly the machines named, or refuses without sending anything. -// -// The all-or-nothing rule push deliberately does NOT follow, and for a reason that holds here and -// not there: a rotation that reached the -// consumer and refused on the provider would leave one end holding a credential the other has -// never heard of — which is the state this whole mechanism exists to make impossible. -func sendTo(ctx context.Context, open *stores, names []string) error { - _, err := sendToEach(ctx, open, names) - return err -} - -// sendToEach is sendTo, answering the machines it sent: those named, and before them the machine -// holding the bus when its user list must go first (novox/hq issue 249) — so a caller that waits for -// the machines it sent waits for that one too. -func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) { - inv := open.inventory - // **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's - // machine for some other module carried the bus's new build with it on 2026-10-06, and the bus - // restarted under every machine with nobody having asked. Refused whole — the send cannot leave the - // bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again. - if !busStepSending(ctx) { - held, err := busHeld(ctx, inv, names) - if err != nil { - return nil, err - } - for _, n := range names { - if why, h := held[n]; h { - return nil, fmt.Errorf("%w: %s", errBusWaits, why) - } - } - } - ident, err := openIdentity(ctx) - if err != nil { - return nil, err - } - defer ident.Close() - - // The private network every declaration of the send is composed with: not computable refuses the send up front. - if _, err := generatorsOf(ctx, open); err != nil { - return nil, err - } - holder, behind, err := brokerBehind(ctx, open, names) - if err != nil { - return nil, err - } - added := "" - if behind && !slices.Contains(names, holder) { - added = holder - } - names = brokerFirst(names, holder, behind) - // **No build moves on a machine without a gate** (novox/hq ADR 0236): a send outside its scope that - // would carry one is refused, said with what waits; the bus's machine added for its user list is left. - if names, err = ungatedIn(ctx, open, names, added); err != nil { - return nil, err - } - - // **A recorded build moves only by a person's push** (novox/hq issue 295, ADR 0245): this send — a - // plan's, a release plan's, a rollback's, a healer's, a rotation's — composes every recorded module at - // the build its machine runs. The bus step is a person's word for the bus alone. - if ctx, err = sendKeeps(ctx, inv); err != nil { - return nil, err - } - - // Held from composing to sending (novox/hq ADR 0100); a caller that holds them already — - // converge, which flips the node and then sends it — is not made to wait on itself. - ctx, release, err := holdNodes(ctx, open, names) - if err != nil { - return nil, err - } - defer release() - - var sending []readyNode - var refusals []string - for _, name := range names { - // Numbered before composing, for the reason composeEach gives (novox/hq issue 204). - numbered, err := allot(ctx, inv, name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - // Read afresh for each machine: the send before it made and recorded things. - composing := freshInput(ctx) - plan, _, err := resolveNode(composing, open, name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - reportUnhostable(name, plan) - reportKept(ctx, plan) - declared, err := composeSend(composing, open, name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - numbered.stamp(&declared) - // **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very - // declaration, the one sent, so nothing composed between a check and the send can slip through. - if everyBuildKept(ctx, name) { - other, err := grantsOnlyIn(ctx, open, name, plan, declared) - if err != nil { - return nil, err - } - if other != "" { - return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other) - } - } - reportLeftOut(name, declared) - sending = append(sending, readyNode{name, declared}) - } - if len(refusals) > 0 { - return nil, fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s", - len(refusals), strings.Join(refusals, "\n\n")) - } - - server, err := connectLink(ctx, nil, nil, nil) - if err != nil { - return nil, err - } - defer server.Close() - - // And every assignment on those machines its membership (novox/hq ADR 0160): composed from the - // same records the bus's accounts are, so what a runtime serves and what its account may are one - // composition. **Issued before the declarations** (novox/hq issue 249): the runtime the - // membership is for arrives with the declaration, and a membership waits for it on the bus; the - // code arriving first was refused its own state until somebody pushed. - if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: " "}, holder, sending); err != nil { - return nil, err - } - sent := make([]string, 0, len(sending)) - for _, s := range sending { - sent = append(sent, s.node) - } - return sent, nil -} - -// issueMemberships publishes the membership of every module on the machines just sent. -// -// Each carries what its module receives and the private network's addresses, from the same -// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is -// given on the bus, and the file written beside it says the same thing. -func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error { - records, err := open.inventory.BusRecords(ctx) - if err != nil { - return err - } - bus, ok := server.Bus().(link.OverNATS) - if !ok { - return nil - } - // Which machines are root-free now (novox/hq ADR 0259 §8): a channel's verified sender is composed for the - // router only from one, beside a router on one. Judged once per push, by the root-free verb's judgement. - var whys map[string]string - records.RootFree, whys = rootVerdictsOf(judgeRootFree(ctx, newRootReader(ctx, open.inventory, bus.Conn), - records.Nodes, time.Now())) - where := broker.PlacementsOf(records, records.Interchangeable) - // What the judgement kept off is said, with its reason (novox/hq issue 930); the self-check gives it back. - for _, line := range droppedVerifiedSenders(records, where, whys) { - fmt.Println(line) - } - // **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The - // raise at start asserts them too, but a module registered and assigned since would otherwise have - // its bucket only after the control plane next restarts — found the first time a module declared - // state: its bundle asked for a bucket that did not exist. Idempotent and cheap. - // - // **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood, - // because the declarations were already away; they are sent after this now — all but the bus's - // own machine, sent before it (deliver) — and a module whose state does not exist is a module - // that fails its start, so they are not sent and the caller tries again rather than reporting the - // rollout done. - buckets, err := open.inventory.DeclaredBuckets(ctx) - if err != nil { - return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err) - } - if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil { - return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err) - } - // **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its - // logs, and a module whose log does not exist fails its first append. - logs, err := open.inventory.DeclaredLogs(ctx) - if err != nil { - return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err) - } - if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil { - return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err) - } - // Every membership is tried, and the first failure named once. - issued := 0 - refused := map[string]error{} - for _, s := range sent { - node := s.node - for _, d := range records.Assigned[node] { - membership := broker.MembershipFor(node, d, where) - membership.Mesh = s.declared.Mesh - for requirement, given := range s.declared.Received[d.Module] { - raw, err := json.Marshal(given) - if err != nil { - return err - } - if membership.Receives == nil { - membership.Receives = map[string]json.RawMessage{} - } - membership.Receives[requirement] = raw - } - body, err := json.Marshal(membership) - if err != nil { - return err - } - if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil { - if refused[node] == nil { - refused[node] = fmt.Errorf("%s: %w", d.Module, err) - } - continue - } - issued++ - } - } - if issued > 0 { - fmt.Printf(" issued %d membership(s)\n", issued) - } - // **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's - // setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again - // as last issued, with only what it hears changed, so nothing else of it moves before its own send. - reheard := reissueHearing(ctx, bus, records, where, sent) - if reheard > 0 { - fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard) - } - // **And says again what it keeps** (novox/hq ADR 0495 clause 2, issue 1790): its word at start is refused until a push - // carries its grant; kept only when it changed, so saying it at every push changes nothing else. - if js := servingBus.Load(); js != nil { - if err := announceOwnStates(ctx, js); err != nil { - fmt.Printf(" states: %v\n", err) - } - } - if len(refused) > 0 { - // Returned, never passed over (novox/hq issue 249): the declarations of the machines they are - // for are not sent, and the rollout that asked is tried again rather than waiting for a push. - return &grantsRefused{nodes: refused} - } - return nil +// listBehind is whether the bus's machine is behind: its have does not name the composition written now, or the +// composition could not be read at all. +func listBehind(digest string, digestErr error, have sdkdelivery.Have, had bool) bool { + return digestErr != nil || !had || have.Composed != digest } // digestOf is what the mesh compares to answer "has this machine been sent what it should be". @@ -1521,70 +415,6 @@ func digestOf(body []byte) string { return hex.EncodeToString(sum[:]) } -// wouldSend is the digest of what each machine should be right now. -// -// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot -// be worked out" is a different problem with a different remedy, and `plan` is where it is said. -func wouldSend(ctx context.Context, open *stores, - nodes []inventory.Node) (map[string]string, error) { - return wouldSendFrom(ctx, open, nodes, nil) -} - -// planned is one machine's plan as planFor answered it, for a caller that already asked. -type planned struct { - plan catalogue.Resolution - settings catalogue.SettingsBy -} - -// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a -// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq -// to-be 45 Phase 0). A machine absent from plans is worked out here. -func wouldSendFrom(ctx context.Context, open *stores, - nodes []inventory.Node, plans map[string]planned) (map[string]string, error) { - - if _, err := generatorsOf(ctx, open); err != nil { - return nil, err - } - var err error - out := map[string]string{} - for _, n := range nodes { - known, have := plans[n.Name] - plan, settings := known.plan, known.settings - if !have { - if plan, settings, err = resolveNode(ctx, open, n.Name); err != nil { - continue - } - } - _, _ = plan, settings - declared, _, err := composeQuestion(ctx, open, n.Name, compose.Reading, false) - if err != nil { - continue - } - // Composed with the number the machine was LAST sent, so this is byte for byte what it was - // sent when nothing else changed. A fresh number here would make every machine read as - // behind for ever (novox/hq 04-ISSUES/107). - if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil { - return nil, err - } - // And the epoch it was last sent under, for the same reason: a new holder of the lease is not a - // change of the machine (novox/hq to-be 45 §6). - if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { - return nil, err - } - // And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment - // elsewhere in the mesh is not a change of this machine. - if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { - return nil, err - } - body, err := declared.Body() - if err != nil { - return nil, err - } - out[n.Name] = digestOf(body) - } - return out, nil -} - // raiseTheBus asserts the streams and consumers the mesh's own traffic needs. // // **Every start, and it says what it did.** The objects are the mesh's, created by nothing else — @@ -1709,141 +539,10 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok return out, nil } -// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107). -// allotting is allot over one inventory, in the shape composeEach takes. -func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) { - return func(node string) (order, error) { return allot(ctx, inv, node) } -} - // epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch // without a bus. var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) } -// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the -// epoch of the lease it is composed under — zero for a machine that has not said it reads one. -type order struct { - sequence int64 - epoch uint64 - // generation is the assignment generation read before the composition, and readsGeneration whether the - // machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts - // under, whether or not the machine is sent it — both kept for the record of the send. - generation int64 - readsGeneration bool - acting uint64 -} - -// allot takes the next sequence for a machine — the number its next declaration carries — under the -// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6). -func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) { - epoch, err := epochForActs(ctx) - if err != nil { - return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err) - } - record, err := inv.NodeByName(ctx, node) - if err != nil { - return order{}, err - } - acting := epoch - if epoch > 0 { - reads, err := inv.ReadsEpoch(ctx, record.ID) - if err != nil { - return order{}, err - } - if !reads { - epoch = 0 - } - } - // **The generation is read here, before the composition reads a single assignment** (novox/hq issue - // 234). A generation only grows, so one read before is never newer than the view composed after it: - // the declaration may claim a generation older than its content, never newer — and a claim newer than - // the content is exactly the stale send the machine must be able to refuse. - generation, err := inv.AssignmentGeneration(ctx) - if err != nil { - return order{}, err - } - readsGeneration, err := inv.ReadsGeneration(ctx, record.ID) - if err != nil { - return order{}, err - } - seq, err := inv.NextSequence(ctx, record.ID) - if err != nil { - return order{}, err - } - return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration, - acting: acting}, nil -} - -// recordSent writes down what a machine was just sent, and returns the digest. -// -// **On a context that outlives the caller's** (novox/hq issue 204). The record is written after the -// declaration is away, so a send that failed is never recorded as current — and a controller being -// replaced mid-send had its context cancelled between the two, so the machine was told and the mesh -// never wrote it down: status read "applied, current" over a machine that had just been sent -// something else. What was sent was sent; the record of it must not depend on the sender living -// another second. Bounded, so a store that is away does not hold a dying process open for ever. -// -// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known: -// what tells a machine held back by a policy or a plan from one a push left behind. -func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte, - builds map[string]string, epoch uint64, sent sentRecord) (string, error) { - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() - record, err := inv.NodeByName(kept, node) - if err != nil { - return "", err - } - digest := digestOf(body) - // The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the - // would-send is never stamped with a generation the machine was not sent; and the send itself, who sent - // it and from which generation, beside them. A record of the send that cannot be written does not make a - // send that is away look failed: it is said, loudly, and the machine's own record stands. - err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{ - Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, - Digest: digest, Modules: sent.modules, Provisioning: sent.plans}) - var unrecorded *inventory.SendNotRecordedError - switch { - case errors.As(err, &unrecorded): - fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+ - "sent it and from which generation could not be written down, so a refusal of it will name no sender "+ - "(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err) - fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err) - case err != nil: - return "", err - } - // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). - // Never a reason to fail a send that is already away: a summary that cannot be kept means the - // next comparison has nothing to hold against, which is how every machine starts. - if summary, err := summarize(body); err == nil { - if raw, err := json.Marshal(summary); err == nil { - if err := inv.RecordSentSummary(kept, record.ID, raw); err != nil { - fmt.Fprintf(os.Stderr, "%s: what it was sent is not kept for comparison: %v\n", node, err) - } - } - } - return digest, nil -} - -// reportUnheldPushed says what a push's machines lack of the seats their modules depend on -// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody -// is looking at, and one line per machine otherwise — a list per machine across the mesh is the -// hundred lines that buried the one that mattered. Nothing for a machine that lacks nothing. -func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[string][]catalogue.Unheld) { - for _, node := range asked { - lines := unheld[node] - if len(lines) == 0 { - continue - } - if named { - fmt.Fprintf(w, "\n%s has %d unmet seat dependenc(ies) (novox/hq ADR 0207):\n", node, len(lines)) - for _, u := range lines { - fmt.Fprintf(w, " %s\n", u) - } - continue - } - fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines)) - } -} - // schemaReach is the highest migration this build carries for the inventory's store (novox/hq issue 352). func schemaReach() (int, error) { migrations, err := inventory.Migrations() @@ -1858,78 +557,3 @@ func schemaReach() (int, error) { } return reach, nil } - -// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose -// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus, -// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said -// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it. -func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements, - sent []readyNode) int { - reached := map[string]bool{} - for _, s := range sent { - reached[s.node] = true - } - issued := 0 - for node, declared := range records.Assigned { - if reached[node] { - continue - } - for _, d := range declared { - if !d.HearsAStream { - continue - } - subject := broker.MembershipSubject(node, d.Module) - last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject) - if err != nil { - if !errors.Is(err, nats.ErrMsgNotFound) { - fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n", - d.Module, node, err) - } - continue - } - body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d)) - if err != nil { - fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n", - d.Module, node, err) - continue - } - if !changed { - continue - } - if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil { - fmt.Printf(" %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err) - continue - } - issued++ - } - } - return issued -} - -// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure. -func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) { - var fields map[string]json.RawMessage - if err := json.Unmarshal(issued, &fields); err != nil { - return nil, false, err - } - var had []broker.HeardIssued - if raw, ok := fields["hears"]; ok { - if err := json.Unmarshal(raw, &had); err != nil { - return nil, false, err - } - } - if slices.Equal(had, hears) { - return issued, false, nil - } - if len(hears) == 0 { - delete(fields, "hears") - } else { - raw, err := json.Marshal(hears) - if err != nil { - return nil, false, err - } - fields["hears"] = raw - } - body, err := json.Marshal(fields) - return body, true, err -} diff --git a/cmd/mesh-controller/push_recreates_test.go b/cmd/mesh-controller/push_recreates_test.go deleted file mode 100644 index d8361730..00000000 --- a/cmd/mesh-controller/push_recreates_test.go +++ /dev/null @@ -1,78 +0,0 @@ -//go:build integration - -package main - -import ( - "bytes" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/link" -) - -// The live case of 2026-10-07, 20:27 UTC: a person's `push` of one machine moved mail to a new build whose -// automx container had a new image, and recreated it; the push's answer said nothing of it. The push now -// says, per machine, every module it moves and what that recreates — before it is sent. A module whose -// build did not change, and one new to the machine, are not moves. -func TestAPushSaysWhatItRecreates(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := func(c string) string { return "registry.invalid:5000/mailu/x@sha256:" + strings.Repeat(c, 64) } - for _, b := range []link.BuildResult{ - aContainerBuild(t, "mailu", "c74f407a", catalogue.PolicyRecord, start, - map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("c"), ""}}), - aContainerBuild(t, "postgres", "c1111111", "", start.Add(time.Second), - map[string][2]string{"server": {image("e"), ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"mailu", "postgres"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"mailu": "c74f407a", "postgres": "c1111111"}); err != nil { - t.Fatal(err) - } - // Mail's new build: automx's image changes, smtp's does not. - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "mailu", "1ab84e69", catalogue.PolicyRecord, - start.Add(time.Minute), map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("d"), ""}})); err != nil { - t.Fatal(err) - } - - // Composed as a person's push composes it. - declared, err := composeSend(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if declared.Builds["mailu"] != "1ab84e69" { - t.Fatalf("a person's push carries mail at %q", declared.Builds["mailu"]) - } - - var said bytes.Buffer - sayWhatAPushRecreates(ctx, open, []readyNode{{node: "anchor", declared: declared}}, &said) - out := said.String() - for _, want := range []string{"anchor moves 1 module(s)", "mailu", "c74f407a → 1ab84e69", - "recreates 1 of mailu's 2 container(s) (with a new image: automx)"} { - if !strings.Contains(out, want) { - t.Fatalf("the push says %q; it does not say %q", out, want) - } - } - if strings.Contains(out, "postgres") { - t.Fatalf("a module whose build did not change is said to move: %q", out) - } - - // A machine never sent anything before has no moves to say. - said.Reset() - sayWhatAPushRecreates(ctx, open, []readyNode{{node: "laptop", declared: declared}}, &said) - if said.Len() != 0 { - t.Fatalf("a machine with no last send is said to move: %q", said.String()) - } -} diff --git a/cmd/mesh-controller/push_test.go b/cmd/mesh-controller/push_test.go deleted file mode 100644 index 541145cd..00000000 --- a/cmd/mesh-controller/push_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "errors" - "strings" - "testing" -) - -// One machine that cannot be worked out is not a reason to leave the mesh unconverged. -// -// A whole-mesh push refused outright the moment any single node failed to resolve, so a module on -// the anchor requiring a provision nobody had assigned a provider for stopped every OTHER machine -// from being sent anything — machines with no relation to the fault, and nothing wrong with them. -// The failure and the punishment were on different machines. -// -// It is the same rule an un-hostable module already follows one level down (a92c11b: one module on -// the wrong machine no longer refuses the whole node), applied one level up. -func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) { - sending, refusals := composeEach( - []string{"anchor", "home-server", "laptop"}, numbered(), - func(node string) (sendable, error) { - if node == "anchor" { - return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`) - } - return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil - }) - - var told []string - for _, s := range sending { - told = append(told, s.node) - } - if strings.Join(told, ",") != "home-server,laptop" { - t.Errorf("a machine with nothing wrong with it was not sent: %v", told) - } - if len(refusals) != 1 || !strings.Contains(refusals[0], "anchor") || - !strings.Contains(refusals[0], "acme-ca") { - t.Errorf("the machine that could not be worked out was not named with its reason: %v", - refusals) - } -} - -// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped -// (novox/hq issue 127): it may have held something before, and only sending the empty -// declaration tells it to drop what the mesh owned. It is never a refusal. -func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) { - sending, refusals := composeEach([]string{"spare"}, numbered(), - func(string) (sendable, error) { return sendable{}, nil }) - if len(sending) != 1 || len(refusals) != 0 { - t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals) - } -} - -// A push that skipped a machine still ends badly, and says what was sent. -// -// **Skipping is not succeeding.** The mesh is not in the state somebody asked for, so the command -// exits non-zero — but it says how many machines it did reach, because the old message ("nothing -// was sent") was the very claim that had become untrue. -func TestASkippedMachineIsStillAnError(t *testing.T) { - if err := couldNotBeResolved(nil, 3); err != nil { - t.Fatalf("a push that resolved every machine reported a problem: %v", err) - } - - err := couldNotBeResolved([]string{"anchor:\nnothing provides \"acme-ca\""}, 2) - if err == nil { - t.Fatal("a push that could not work out a machine reported success") - } - said := err.Error() - if strings.Contains(said, "nothing was sent") { - t.Errorf("the push says nothing was sent, and it sent two machines: %q", said) - } - for _, want := range []string{"anchor", "acme-ca", "2 other node(s) were"} { - if !strings.Contains(said, want) { - t.Errorf("the refusal does not say %q: %q", want, said) - } - } -} - -// numbered is an allotter for tests: one higher per call, as the inventory's is per machine. -func numbered() func(string) (order, error) { - var n int64 - return func(string) (order, error) { n++; return order{sequence: n}, nil } -} diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index 3d037c68..c71d9b21 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -42,9 +42,6 @@ type meshStatus struct { // whose is older is still working — and Waiting cannot tell those apart, because the sent // digest is recorded at send, not at apply. Reported []machineReported `json:"reported"` - // Plans is what the last merges produced and where each stands (novox/hq ADR 0162): the - // open ones first, each saying its tier, what it waits for, and whether it has waited too long. - Plans []planStatus `json:"plans"` // Unresolved is every machine that cannot be worked out at all, with what the mesh said when // it tried. **A machine here is in none of the lists above**: nothing was computed for it, so // there is nothing to compare it against and nothing it can be behind — which is why a @@ -114,33 +111,6 @@ type meshStatus struct { // why they could not be read. Pending []pendingStatus `json:"pending,omitempty"` PendingUnread string `json:"pendingUnread,omitempty"` - // LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which - // assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485). - // A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read. - LastSent []machineLastSent `json:"lastSent"` - LastSentUnread string `json:"lastSentUnread,omitempty"` -} - -// machineLastSent is one machine's last send as status says it. -type machineLastSent struct { - Node string `json:"node"` - Sequence int64 `json:"sequence"` - SentAt time.Time `json:"sentAt"` - Sender string `json:"sender"` - Epoch int64 `json:"epoch"` - Generation int64 `json:"generation"` - Digest string `json:"digest"` - Modules []string `json:"modules"` - // Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it - // has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed. - Answer string `json:"answer"` - Refused string `json:"refused,omitempty"` - Failed int `json:"failed,omitempty"` - AnsweredAt *time.Time `json:"answeredAt,omitempty"` - // RefusedForGeneration is when the machine refused it for coming from an older assignment generation than - // it applied, and AppliedGeneration that generation; absent when it did not. - RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"` - AppliedGeneration int64 `json:"appliedGeneration,omitempty"` } // pendingStatus is one pending assignment as status says it. @@ -252,7 +222,7 @@ func statusAsJSON(asked answers) ([]byte, error) { out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{}, Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}, Reported: []machineReported{}, Unresolved: []machineUnresolved{}, - Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.tierBounds), + Network: asked.network, Adopted: adoptedNodes(nodes), NotAnnounced: asked.notAnnounced, NotAnnouncedWhy: asked.notAnnouncedWhy} // In a stated order, so two readings of an unchanged mesh are the same document. untakenNodes := make([]string, 0, len(asked.untaken)) @@ -300,7 +270,6 @@ func statusAsJSON(asked answers) ([]byte, error) { out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note}) } - out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread for name := range asked.refused { out.Unresolved = append(out.Unresolved, machineUnresolved{ Node: name, Problem: asked.refused[name]}) @@ -357,20 +326,3 @@ func say(value any) error { fmt.Println(string(body)) return nil } - -// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485). -func lastSentStatus(sends map[string]inventory.Send) []machineLastSent { - out := []machineLastSent{} - for _, s := range sends { - modules := s.Modules - if modules == nil { - modules = []string{} - } - out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender, - Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome, - Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At, - RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) - return out -} diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go deleted file mode 100644 index 7e16c74e..00000000 --- a/cmd/mesh-controller/recorded_kept.go +++ /dev/null @@ -1,219 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "sort" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245). -// -// **A send carries the machine's whole declaration** (ADR 0221), composed from the build the mesh holds -// of every module on it. A module whose upgrade policy records — postgres, mongodb, keycloak, the -// network path — has its new build registered at its merge and sent nowhere, "until a person pushes". -// But every other send to its machine composed it too: on 2026-10-07 a catalogue merge adopting the -// images' health checks rebuilt postgres and mongodb with the rest, and the plan's gated send to the -// control node for mail — and to the anchor for the spreadsheet app — carried both, recreating the -// providers every consumer on those machines drops with. No gate judged them (the gate judges only what -// rolls out), and nobody had pushed. -// -// So **every send but a person's push composes a recorded module at the build its machine was last -// sent**: the manifest of that build, from the build records, in place of the one the mesh holds. The -// machine runs what it ran; the send records that it still carries that build; `status` keeps saying -// the machine is behind, and `push ` — a person's word — sends the new one. The bus step is a -// person's too, and carries the bus; any other recorded module waiting on the bus's machine stays. -// -// A recorded module the machine was never sent (a new assignment) is composed as the mesh holds it — -// there is nothing running to keep. One whose kept build is no longer in the records refuses the send, -// said: composing the new build would be the very move this exists to stop. - -type keepRecordedKey struct{} - -// sendKeeps is the context a send that is not a person's push composes under: every recorded module -// kept at the build its machine runs — except, on the bus step, the bus. -func sendKeeps(ctx context.Context, inv *inventory.Inventory) (context.Context, error) { - if !busStepSending(ctx) { - return keepingRecorded(ctx), nil - } - bus, err := pendingBus(ctx, inv) - if err != nil { - return nil, err - } - return keepingRecorded(ctx, bus.module), nil -} - -// keepingRecorded is a context whose sends compose every recorded module at the build its machine runs, -// except the modules named (the bus, on the bus step). -func keepingRecorded(ctx context.Context, except ...string) context.Context { - skip := map[string]bool{} - for _, m := range except { - skip[m] = true - } - return context.WithValue(ctx, keepRecordedKey{}, skip) -} - -// keptExcept is whether this context keeps recorded modules, and the modules it lets move. -func keptExcept(ctx context.Context) (map[string]bool, bool) { - skip, on := ctx.Value(keepRecordedKey{}).(map[string]bool) - return skip, on -} - -type keepEveryBuildKey struct{} - -// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out — -// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine -// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate -// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses -// what the change newly grants. -// -// **That machine alone.** Composing one machine resolves the others too — who is on the private network, -// who answers a requirement — on the same context, and those are no part of the step's send: they are -// composed as any send composes them. Kept for every machine, a machine whose last send is not known -// refused the bus's machine's composition. -func keepingEveryBuild(ctx context.Context, node string) context.Context { - return context.WithValue(ctx, keepEveryBuildKey{}, node) -} - -// everyBuildKept is whether this context keeps every module of this machine at the build it runs. -func everyBuildKept(ctx context.Context, node string) bool { - on, _ := ctx.Value(keepEveryBuildKey{}).(string) - return on != "" && on == node -} - -type keepForSettingsKey struct{} - -// keepingBuildsForSettings is a context whose sends compose every module of one machine — recorded or rolling out — -// at the build that machine was last sent, as keepingEveryBuild does, for a settings walk's send (novox/hq ADR 0344): -// the layer the operator approved, and nothing of any module's new code. A build held for a person, or waiting -// for a gate, stays where it is; the walk that is its own carries it. -func keepingBuildsForSettings(ctx context.Context, node string) context.Context { - return context.WithValue(ctx, keepForSettingsKey{}, node) -} - -// errSettingsCannotKeep is a settings walk's send refused because a build of its machine cannot be kept: what the -// machine was last sent is not known, or a build it runs is gone from the records. It does not clear by itself, so -// the walk fails on it rather than trying again (ADR 0344). -var errSettingsCannotKeep = errors.New("the settings cannot be sent with every build kept") - -// settingsKeptOn is whether this context is a settings walk's send to this machine. -func settingsKeptOn(ctx context.Context, node string) bool { - on, _ := ctx.Value(keepForSettingsKey{}).(string) - return on != "" && on == node -} - -// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a -// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context -// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere — -// ADR 0221). -// -// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue -// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and -// composing the mesh's builds would be the very move the grants step must not make. -func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) { - every := everyBuildKept(ctx, node) || settingsKeptOn(ctx, node) - skip, on := keptExcept(ctx) - if !on && !every { - return nil, nil - } - if every { - skip = nil - } - inv := open.inventory - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - if !known { - if settingsKeptOn(ctx, node) { - return nil, fmt.Errorf("%w: what %s was last sent is not known (novox/hq ADR 0344)", errSettingsCannotKeep, node) - } - if every { - return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+ - "there alone with every build kept (novox/hq issue 490)", node) - } - return nil, nil - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - var f *moveFacts - out := map[string]string{} - for m, was := range sent { - now, held := current[m] - if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) { - continue - } - if f == nil { - read, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - f = &read - } - if f.identical(m, was, now.Commit) { - continue - } - // **Under its own `record`, a move that recreates nothing and moves no data is not kept** (novox/hq ADR 0377 - // §1): it goes as a module that rolls out goes. A person's `record`, the bus, and a move that cannot be - // judged are kept as before (§4, §5). - if !every && f.rolls(m, was) { - continue - } - out[m] = was - } - // **And the controller and the node-engine at the build the machine was last sent, over an older record** - // (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already. - if !every { - core, err := coreKept(ctx, inv, node, sent, current) - if err != nil { - return nil, err - } - for m, was := range core { - out[m] = was - } - } - return out, nil -} - -// applyKept puts the manifest of each module kept at the build its machine runs into a shelf, and refuses, in the words of -// keepRecorded, a build the records no longer hold. The adapter to the composition's input (compose_input.go) calls it with -// a scratch shelf, to read the manifests it gives the node as its own Kept. -func applyKept(ctx context.Context, open *stores, node string, kept map[string]string, shelf map[string]catalogue.Manifest) error { - names := make([]string, 0, len(kept)) - for m := range kept { - names = append(names, m) - } - sort.Strings(names) - for _, m := range names { - ran, found, err := open.inventory.ManifestAt(ctx, m, kept[m]) - if err != nil { - return err - } - if !found && settingsKeptOn(ctx, node) { - return fmt.Errorf("%w: %s runs %s's build %s, which the build records no longer hold (novox/hq ADR 0344)", - errSettingsCannotKeep, node, m, short(kept[m])) - } - if !found && everyBuildKept(ctx, node) { - return fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+ - "list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m])) - } - if !found && guarded(coreComponent(m)) { - return fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+ - "no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+ - "a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)", - node, m, short(kept[m]), node, m) - } - if !found { - return fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+ - "records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+ - "one on a person's word (novox/hq ADR 0245)", m, node, short(kept[m]), node) - } - shelf[m] = ran - } - return nil -} diff --git a/cmd/mesh-controller/recorded_kept_integration_test.go b/cmd/mesh-controller/recorded_kept_integration_test.go deleted file mode 100644 index 8f96201e..00000000 --- a/cmd/mesh-controller/recorded_kept_integration_test.go +++ /dev/null @@ -1,140 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) - mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {pgImage, ""}}), - aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second), - map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"postgres", "mailu"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil { - t.Fatal(err) - } - // The merge: both adopt a health check; the images are the same. - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), - map[string][2]string{"server": {pgImage, "runtime"}}), - aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second), - map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}), - } { - if _, _, err := takeIn(ctx, inv, b); err != nil { - t.Fatal(err) - } - } - - healthOf := func(plan catalogue.Resolution, module, id string) any { - for _, m := range plan.Modules { - for _, r := range m.Resources { - if m.Module == module && r["id"] == id { - return r["health"] - } - } - } - return nil - } - - // A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves. - kept := keepingRecorded(ctx) - plan, _, err := resolveNode(kept, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") != nil { - t.Fatal("a send that is not a person's push composed the recorded module's new build") - } - if healthOf(plan, "mailu", "smtp") == nil { - t.Fatal("the module that rolls out was not composed at its new build") - } - if imageOf(t, plan, "postgres", "server") != pgImage { - t.Fatal("the recorded module's container lost its image") - } - declared, err := composeSend(kept, open, "anchor") - if err != nil { - t.Fatal(err) - } - if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" { - t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds) - } - - // A person's push: the recorded module's new build. - plan, _, err = resolveNode(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("a person's push did not compose the recorded module's new build") - } - - // The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay. - plan, _, err = resolveNode(keepingRecorded(ctx, "postgres"), open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("the module a send is let move was kept") - } - - // What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves. - if under, err := sendKeeps(ctx, inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || len(skip) != 0 { - t.Fatalf("an ordinary send keeps %v %v", on, skip) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2", - Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if under, err := sendKeeps(withBusStep(ctx), inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 { - t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip) - } - - // And a recorded module whose kept build the records no longer hold refuses the send, said. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil { - t.Fatal(err) - } - if _, _, err := resolveNode(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { - t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err) - } - - // And the gated send for mail says what it recreates: both containers, no new image, at once. - moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}} - sayRecreations(ctx, open, moves) - if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") { - t.Fatalf("the send says %q of mail's containers", moves[0].Recreates) - } - if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") { - t.Fatalf("the plan's note says %q", said) - } -} diff --git a/cmd/mesh-controller/recorded_kept_test.go b/cmd/mesh-controller/recorded_kept_test.go deleted file mode 100644 index 27b6fab1..00000000 --- a/cmd/mesh-controller/recorded_kept_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/link" -) - -// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245). - -// aContainerBuild is a build outcome of a module of containers, each named by id with the image and the -// health it is given; a policy when one is said. -func aContainerBuild(t *testing.T, module, commit, policy string, asked time.Time, containers map[string][2]string) link.BuildResult { - t.Helper() - var resources []any - for id, c := range containers { - r := map[string]any{"id": id, "type": "container", "name": module + "-" + id, "image": c[0]} - if c[1] != "" { - r["health"] = map[string]any{"kind": c[1]} - } - resources = append(resources, r) - } - m := map[string]any{"module": module, "version": "1", "resources": resources} - if policy != "" { - m["upgrade"] = map[string]any{"policy": policy, "why": "a provider whose restart drops every consumer"} - } - manifest, _ := json.Marshal(m) - return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module, - On: "anchor", Module: module, Commit: commit, Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} -} - -// imageOf is the image the composed plan of a machine gives one of a module's containers. -func imageOf(t *testing.T, plan catalogue.Resolution, module, id string) string { - t.Helper() - for _, m := range plan.Modules { - if m.Module != module { - continue - } - for _, r := range m.Resources { - if r["id"] == id { - image, _ := r["image"].(string) - return image - } - } - } - t.Fatalf("%s has no container %s in the plan", module, id) - return "" -} - -// Tonight's case, 2026-10-07: a catalogue merge adopting the images' own health checks rebuilt the -// database (policy record) with mail (policy roll). The plan's gated send for mail carried the -// database's new build to the control node and recreated it, unjudged, with nobody's word. A send that -// is not a person's push now composes the database at the build the machine runs and records that it -// still carries it; a person's push composes the new one; the bus step moves the bus alone. - -// The send says what it recreates (ADR 0245): mail's health checks adopted recreate both its -// containers, with no new image, every one at once. -func TestASendSaysWhatItRecreates(t *testing.T) { - image := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) - from := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{ - {"id": "smtp", "type": "container", "image": image}, - {"id": "imap", "type": "container", "image": image}, - {"id": "redis", "type": "container", "image": image}, - {"id": "config", "type": "file", "path": "/etc/x"}}} - to := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{ - {"id": "smtp", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}}, - {"id": "imap", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}}, - {"id": "redis", "type": "container", "image": image}, - {"id": "config", "type": "file", "path": "/etc/y"}}} - r := catalogue.Recreates(from, to) - if !r.SpecOnly() || len(r.Recreated) != 2 || r.Containers != 3 { - t.Fatalf("recreation %+v", r) - } - said := r.Say("mailu") - for _, want := range []string{"recreates 2 of mailu's 3", "no new image", "imap, smtp", "interrupted"} { - if !strings.Contains(said, want) { - t.Fatalf("%q does not say %q", said, want) - } - } - if catalogue.Recreates(from, from).Say("mailu") != "" { - t.Fatal("a move that recreates nothing said something") - } - to.Resources[2]["image"] = strings.Replace(image, "c", "d", 1) - if r := catalogue.Recreates(from, to); r.SpecOnly() || len(r.Images) != 1 { - t.Fatalf("a new image read as a declaration only: %+v", r) - } -} diff --git a/cmd/mesh-controller/recorded_moves.go b/cmd/mesh-controller/recorded_moves.go deleted file mode 100644 index be6eb18f..00000000 --- a/cmd/mesh-controller/recorded_moves.go +++ /dev/null @@ -1,235 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR -// 0377, changing ADR 0242 §1 and ADR 0360 §1). -// -// A module records because its manifest says so, because it keeps irreplaceable data, because a person set it, or -// because it provides the bus. The first two hold for a reason the manifest shows — a change that recreates mail's -// containers interrupts everyone's mail, a new database image may upgrade its databases in place — and that hold -// also held the moves that interrupt nothing: on 2026-10-15 the database server moved with only its tools bundle -// changed, its walk said it "recreates none of its containers", and ace stayed behind until a person pushed. -// -// So **a move under a module's own `record` is judged per machine, against the build that machine was last sent** -// (§2): when it recreates none of the module's containers and moves none of its data (catalogue.Holds, §3) it -// goes as a module that rolls out goes — a walk sends it, and every other send treats it as a move waiting for its -// walk — and otherwise it is kept at the build the machine runs, as before. The module's policy stays `record`. A -// move that cannot be judged waits (§4); a person's `record` and the bus hold every move (§5); and each walk says -// per machine which way a recorded module went (§6). - -// recordedRollsWhy is what a walk says of a recorded module's move that went out (ADR 0377 §6). -const recordedRollsWhy = "went out: it recreates none of its containers and moves none of its data (novox/hq ADR 0377)" - -// recordVerdict is one machine's move of a recorded module: whether it rolls out, and what the walk says of it. -type recordVerdict struct { - rolls bool - // said is what the walk says: recordedRollsWhy, or "waits: " and why — the recreate words, the data change - // named, the person's or the bus's hold, or that it could not be judged. - said string -} - -// recordedMove judges a module's move on a machine from the build it was last sent (`was`) to the one it would be -// sent now. A module that rolls out rolls; one whose policy is a person's or the bus's waits, said; one under its -// own `record` rolls when the move recreates nothing and moves no data, and waits when it does or cannot be judged. -func (f moveFacts) recordedMove(module, was string) recordVerdict { - now := f.current[module] - switch { - case now.RollOut: - return recordVerdict{rolls: true} - case now.From == catalogue.FromPerson: - return recordVerdict{said: "waits: a person set it to record, and holds every move of it until they push (novox/hq ADR 0377 §5)"} - case now.From == catalogue.FromBus: - return recordVerdict{said: "waits: it provides the bus, which moves only by the bus step a person starts (novox/hq ADR 0377 §5)"} - case !now.OwnRecord(): - return recordVerdict{said: "waits: its policy records"} - case f.judge == nil: - return recordVerdict{said: "waits: it could not be judged — the build records were not read (novox/hq ADR 0377 §4)"} - } - return f.judge(module, was, now.Commit) -} - -// rolls is whether a module's move on a machine from `was` goes as a move that rolls out (ADR 0377 §1). -func (f moveFacts) rolls(module, was string) bool { - return f.recordedMove(module, was).rolls -} - -// recordJudge is the judging of moves under a module's own `record`, reading the build records once per move and -// the catalogue once (ADR 0377 §2–§4). -func recordJudge(ctx context.Context, inv *inventory.Inventory) func(module, was, now string) recordVerdict { - judged := map[string]recordVerdict{} - var shelf map[string]catalogue.Manifest - var shelfErr error - shelfRead := false - return func(module, was, now string) recordVerdict { - key := module + "\x00" + was + "\x00" + now - if v, seen := judged[key]; seen { - return v - } - v := func() recordVerdict { - cannot := func(why string) recordVerdict { - return recordVerdict{said: "waits: it could not be judged — " + why + " (novox/hq ADR 0377 §4)"} - } - if was == "" { - return cannot("what the machine was last sent of it is not known") - } - from, found, err := inv.ManifestAt(ctx, module, was) - if err != nil { - return cannot(fmt.Sprintf("its build %s, which the machine was last sent, cannot be read: %v", short(was), err)) - } - if !found { - return cannot(fmt.Sprintf("its build %s, which the machine was last sent, is not in the build records", short(was))) - } - if !shelfRead { - shelf, shelfErr = inv.Catalogue(ctx) - shelfRead = true - } - if shelfErr != nil { - return cannot(fmt.Sprintf("the build it would be sent cannot be read: %v", shelfErr)) - } - to, held := shelf[module] - if !held { - return cannot("the build it would be sent is not in the catalogue") - } - if why := catalogue.Holds(from, to); why != "" { - return recordVerdict{said: "waits: " + why} - } - return recordVerdict{rolls: true, said: recordedRollsWhy} - }() - judged[key] = v - return v - } -} - -// judgeRecordedInWalk judges, for a walk carrying a module whose policy records, each machine running it whose -// build would move (ADR 0377 §2), keeps on the plan which way each went (§6), says it on the controller's log, and -// answers the machines its move goes out to. Judged once per walk: a walk read again answers what it kept. -func judgeRecordedInWalk(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, module string, - state *inventory.PlanModule) ([]string, error) { - if state.Recorded != nil { - return wentOut(state.Recorded), nil - } - running, err := inv.Running(ctx, module) - if err != nil { - return nil, err - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - now := f.current[module] - said := map[string]string{} - for _, n := range running { - sent, known, err := inv.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - was, carried := sent[module] - if known && carried && f.identical(module, was, now.Commit) { - continue // no move there - } - v := f.recordedMove(module, was) - if !known { - v = recordVerdict{said: "waits: it could not be judged — what " + n + " was last sent is not known (novox/hq ADR 0377 §4)"} - } - said[n] = v.said - fmt.Printf("%s: %s on %s %s\n", p.ID, module, n, v.said) - } - if len(said) > 0 { - state.Recorded = said - } - return wentOut(said), nil -} - -// wentOut is the machines a recorded module's move went out to, sorted. -func wentOut(recorded map[string]string) []string { - var out []string - for _, n := range sortedKeysOf(recorded) { - if recorded[n] == recordedRollsWhy { - out = append(out, n) - } - } - return out -} - -// recordedSaid is, for a walk's note at its end, which way each recorded module it carried went on each machine -// (ADR 0377 §6). -func recordedSaid(p inventory.Plan) []string { - var out []string - for _, m := range sortedKeysOf(planModuleNames(p)) { - s := p.Modules[m] - for _, n := range sortedKeysOf(s.Recorded) { - out = append(out, fmt.Sprintf("%s on %s %s", m, n, s.Recorded[n])) - } - } - return out -} - -func planModuleNames(p inventory.Plan) map[string]string { - out := map[string]string{} - for m, s := range p.Modules { - if s != nil && len(s.Recorded) > 0 { - out[m] = m - } - } - return out -} - -// heldMove is one move of a recorded module that waits for a person on one machine (ADR 0377 §6). -type heldMove struct { - Module string - From, To string - // Why is the walk's own word for it: "waits: " and the recreate words, the data change, or the hold. - Why string - // Chosen is whether a person's own choice holds it — a module they set to record, or the bus's step — which the - // operator is not asked about again: they chose that hold, and the bus has its own condition (S17). - Chosen bool -} - -// heldMovesOn is, on one machine, every move of a recorded module that waits for a person, in module order. -func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]heldMove, error) { - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known { - return nil, err - } - var out []heldMove - for _, m := range sortedKeysOf(sent) { - was := sent[m] - now, held := f.current[m] - if !held || now.RollOut || now.Commit == "" || was == "" || f.identical(m, was, now.Commit) { - continue - } - if v := f.recordedMove(m, was); !v.rolls { - out = append(out, heldMove{Module: m, From: was, To: now.Commit, Why: v.said, - Chosen: now.From == catalogue.FromPerson || now.From == catalogue.FromBus}) - } - } - return out, nil -} - -// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why -// (ADR 0377 §6): what `status` names the machine behind for, with `push ` as the remedy. A move under a -// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk. -func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) { - moves, err := heldMovesOn(ctx, inv, f, node) - var out []string - for _, h := range moves { - out = append(out, fmt.Sprintf("%s %s → %s %s", h.Module, short(h.From), short(h.To), h.Why)) - } - return out, err -} - -// stringsKeys is a map of lists as a map of its keys, for sortedKeysOf. -func stringsKeys(m map[string][]string) map[string]string { - out := map[string]string{} - for k := range m { - out[k] = k - } - return out -} diff --git a/cmd/mesh-controller/recorded_moves_test.go b/cmd/mesh-controller/recorded_moves_test.go deleted file mode 100644 index 64b705e3..00000000 --- a/cmd/mesh-controller/recorded_moves_test.go +++ /dev/null @@ -1,363 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR -// 0377). Seen 2026-10-15: the database server moved with only its tools bundle changed, its walk said it -// "recreates none of its containers", and ace stayed behind until a person pushed. - -// aMailu is a mail module as the catalogue declares it, small: it records because its manifest says so, keeps its -// mailboxes as data, mounts its environment file, and serves its tools from a bundle. -func aMailu(policy bool) catalogue.Manifest { - raw := `{"module":"mailu","version":"1","tools":["mailu_status"], - "data":{"own":[{"id":"mail","path":"${dir:data-mail}","class":"valuable","why":"every mailbox"}]}, - "bundles":[{"name":"tools","source":"store/mailu-tools@sha256:aaa","digest":"sha256:aaa"}], - "resources":[ - {"id":"state","type":"directory"}, - {"id":"mesh-state","type":"directory"}, - {"id":"data-mail","type":"directory"}, - {"id":"config-env","type":"file","path":"${dir:state}/mailu.env","content":"A=1"}, - {"id":"runtime-config","type":"file","path":"${dir:mesh-state}/config.json","content":"{}"}, - {"id":"imap","type":"container","name":"mailu-imap","image":"registry.invalid/imap@sha256:1", - "volumes":["${dir:data-mail}:/mail"],"env-file":["${dir:state}/mailu.env"]}]}` - var m catalogue.Manifest - if err := json.Unmarshal([]byte(raw), &m); err != nil { - panic(err) - } - if policy { - m.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "people's mail: a person chooses the moment"} - } - return m -} - -func resourceOf(m *catalogue.Manifest, id string) map[string]any { - for _, r := range m.Resources { - if r["id"] == id { - return r - } - } - panic("no resource " + id) -} - -// recordedMesh is a mesh running mailu at build c1 on anchor and laptop, as each was last sent. -type recordedMesh struct { - open *stores - sent [][]string -} - -func aRecordedMesh(t *testing.T, c1 catalogue.Manifest) *recordedMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - rm := &recordedMesh{open: open} - rm.register(t, c1, "c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, c1.Module); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{c1.Module: "c1"}); err != nil { - t.Fatal(err) - } - } - return rm -} - -// register keeps a build of a module and makes it the mesh's. -func (rm *recordedMesh) register(t *testing.T, m catalogue.Manifest, commit string, at time.Time) { - t.Helper() - ctx := t.Context() - raw, _ := json.Marshal(m) - if err := rm.open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module + "-" + commit, Module: m.Module, - Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + m.Module, Manifest: raw, Asked: at, At: at}); err != nil { - t.Fatal(err) - } - if err := rm.open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + m.Module, BuiltFrom: commit, Head: commit, Asked: at}); err != nil { - t.Fatal(err) - } -} - -// keptOn is what an ordinary send — a walk's, a healer's — keeps on a machine: module → the build it keeps. -func (rm *recordedMesh) keptOn(t *testing.T, node string) map[string]string { - t.Helper() - ctx := t.Context() - under, err := sendKeeps(ctx, rm.open.inventory) - if err != nil { - t.Fatal(err) - } - kept, err := recordedKept(under, rm.open, node) - if err != nil { - t.Fatal(err) - } - return kept -} - -// walk opens a merge's walk of mailu's c2, built, and walks it with every send composed as an ordinary send -// composes it — a recorded module kept where its move waits — and applied at once. -func (rm *recordedMesh) walk(t *testing.T) inventory.Plan { - t.Helper() - ctx := t.Context() - inv := rm.open.inventory - wasSend := sendRollout - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - rm.sent = append(rm.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - kept := rm.keptOn(t, node) - builds := map[string]string{"mailu": current["mailu"].Commit} - if was, held := kept["mailu"]; held { - builds["mailu"] = was - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := inv.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { - return nil, err - } - if _, err := inv.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-mailu", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"mailu"}}, - Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-mailu-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - for i := 0; i < 8; i++ { - advancePlans(ctx, rm.open) - } - p, err := inv.PlanByID(ctx, "plan-mailu") - if err != nil { - t.Fatal(err) - } - return p -} - -func (rm *recordedMesh) sentBuild(t *testing.T, node string) string { - t.Helper() - sent, _, err := rm.open.inventory.SentBuilds(t.Context(), node) - if err != nil { - t.Fatal(err) - } - return sent["mailu"] -} - -// recordedWaitsOn is what `status` names a machine behind for, waiting for a person (ADR 0377 §6). -func (rm *recordedMesh) recordedWaitsOn(t *testing.T, node string) []string { - t.Helper() - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - held, err := recordedWaiting(t.Context(), rm.open.inventory, f, node) - if err != nil { - t.Fatal(err) - } - return held -} - -// §1, §6: a module that records because its manifest says so, whose move changes only its tools bundle, goes out -// with its walk to every machine, without a push; the walk says per machine that it went out and why; the policy -// still reads record; and status names no machine behind for it. -func TestARecordedModulesBundleOnlyMoveRollsOutInItsWalk(t *testing.T) { - c1 := aMailu(true) - rm := aRecordedMesh(t, c1) - c2 := aMailu(true) - c2.Bundles[0].Digest, c2.Bundles[0].Source = "sha256:bbb", "store/mailu-tools@sha256:bbb" - c2.Tools = append(c2.Tools, "mailu_queue") - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - - for _, n := range []string{"anchor", "laptop"} { - if _, kept := rm.keptOn(t, n)["mailu"]; kept { - t.Fatalf("an ordinary send to %s keeps mailu at c1 for a move that recreates nothing", n) - } - } - p := rm.walk(t) - if p.State != inventory.PlanDone { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - for _, n := range []string{"anchor", "laptop"} { - if got := rm.sentBuild(t, n); got != "c2" { - t.Fatalf("%s was sent mailu %s; want c2, with its walk", n, got) - } - if said := p.Modules["mailu"].Recorded[n]; said != recordedRollsWhy { - t.Fatalf("the walk says of mailu on %s %q", n, said) - } - if held := rm.recordedWaitsOn(t, n); len(held) != 0 { - t.Fatalf("status names %s behind for %v", n, held) - } - } - if !strings.Contains(p.Note, "mailu on anchor went out") || !strings.Contains(p.Note, "mailu on laptop went out") { - t.Fatalf("the walk's note does not say which way mailu went: %q", p.Note) - } - if u, err := rm.open.inventory.UpgradeOf(t.Context(), "mailu"); err != nil || u.Policy() != catalogue.PolicyRecord { - t.Fatalf("mailu's policy is %q (%v); want record still", u.Policy(), err) - } -} - -// §1, §2, §6: a move that recreates a container — a new image, or a changed declaration — still waits for a push; -// the walk says per machine that it waits, with the recreate words; and status names the machine behind, with -// `push ` as the remedy. A bundle-only change on top of it stays held: it is judged against what the machine -// was last sent. -func TestARecordedModulesMoveThatRecreatesAContainerWaits(t *testing.T) { - for name, change := range map[string]func(m *catalogue.Manifest){ - "image": func(m *catalogue.Manifest) { resourceOf(m, "imap")["image"] = "registry.invalid/imap@sha256:2" }, - "spec": func(m *catalogue.Manifest) { resourceOf(m, "imap")["health"] = map[string]any{"kind": "runtime"} }, - } { - t.Run(name, func(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - change(&c2) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - p := rm.walk(t) - for _, n := range []string{"anchor", "laptop"} { - if got := rm.sentBuild(t, n); got != "c1" { - t.Fatalf("%s was sent mailu %s; want c1 kept, waiting for a push", n, got) - } - if said := p.Modules["mailu"].Recorded[n]; !strings.HasPrefix(said, "waits: recreates 1 of mailu's 1") { - t.Fatalf("the walk says of mailu on %s %q", n, said) - } - if held := rm.recordedWaitsOn(t, n); len(held) != 1 || !strings.Contains(held[0], "recreates") { - t.Fatalf("status names %s behind for %v", n, held) - } - } - // §2: the next build changes only the bundle, and the machines were last sent c1: still held. - c3 := c2 - c3.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/mailu-tools@sha256:ccc", Digest: "sha256:ccc"}} - rm.register(t, c3, "c3", time.Now()) - if kept := rm.keptOn(t, "anchor"); kept["mailu"] != "c1" { - t.Fatalf("a bundle-only build on top of a held container change is not held: %v", kept) - } - }) - } -} - -// §3: a move touching a file a container mounts, a directory holding declared data, the data declaration or a -// provider waits. -func TestARecordedModulesMoveThatTouchesDataOrAProviderWaits(t *testing.T) { - for name, c := range map[string]struct { - change func(m *catalogue.Manifest) - says string - }{ - "a mounted file": {func(m *catalogue.Manifest) { resourceOf(m, "config-env")["content"] = "A=2" }, "which its container imap mounts"}, - "a data directory": {func(m *catalogue.Manifest) { resourceOf(m, "data-mail")["mode"] = "0700" }, "data-mail"}, - "the data declaration": {func(m *catalogue.Manifest) { - m.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Path: "${dir:data-mail}", Class: "irreplaceable", Why: "x"}}} - }, "data declaration"}, - "a provider": {func(m *catalogue.Manifest) { m.Requires = append(m.Requires, "postgres-database") }, "what it requires"}, - } { - t.Run(name, func(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - c.change(&c2) - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - kept := rm.keptOn(t, "anchor") - if kept["mailu"] != "c1" { - t.Fatalf("an ordinary send does not keep mailu at c1: %v", kept) - } - held := rm.recordedWaitsOn(t, "anchor") - if len(held) != 1 || !strings.Contains(held[0], c.says) { - t.Fatalf("status says %v; want it to name %q", held, c.says) - } - }) - } -} - -// §5: a module a person set to record, and the bus, wait on a bundle-only move. -func TestAPersonsRecordAndTheBusHoldEveryMove(t *testing.T) { - t.Run("a person sets record", func(t *testing.T) { - rm := aRecordedMesh(t, aMailu(false)) - c2 := aMailu(false) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventory.Upgrade{Why: "the migration week", By: "jochen"}); err != nil { - t.Fatal(err) - } - p := rm.walk(t) - if got := rm.sentBuild(t, "anchor"); got != "c1" { - t.Fatalf("a person's record was walked past: anchor was sent %s", got) - } - if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "a person set it to record") { - t.Fatalf("the walk says %q", said) - } - if held := rm.recordedWaitsOn(t, "anchor"); len(held) != 1 { - t.Fatalf("status names anchor behind for %v", held) - } - }) - t.Run("the bus", func(t *testing.T) { - bus := aMailu(false) - bus.Module = "nats" - bus.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - bus.Data = nil - rm := aRecordedMesh(t, bus) - c2 := bus - c2.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/nats-tools@sha256:bbb", Digest: "sha256:bbb"}} - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - if kept := rm.keptOn(t, "anchor"); kept["nats"] != "c1" { - t.Fatalf("the bus moved by an ordinary send: %v", kept) - } - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - if v := f.recordedMove("nats", "c1"); v.rolls || !strings.Contains(v.said, "bus") { - t.Fatalf("the bus's move: %+v", v) - } - }) -} - -// §4: a move whose earlier build is not in the build records cannot be judged, waits, and is said so. -func TestARecordedMoveThatCannotBeJudgedWaits(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - // anchor was last sent a build the records do not hold. - if err := rm.open.inventory.RecordSent(t.Context(), nodeID(t, rm.open, "anchor"), "d-anchor-c0", - map[string]string{"mailu": "c0"}); err != nil { - t.Fatal(err) - } - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - if v := f.recordedMove("mailu", "c0"); v.rolls || !strings.Contains(v.said, "could not be judged") || - !strings.Contains(v.said, "not in the build records") { - t.Fatalf("an unjudgeable move: %+v", v) - } - if _, _, err := resolveNode(keepingRecorded(t.Context()), rm.open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { - t.Fatalf("a send that cannot keep the unjudged build did not refuse with its remedy: %v", err) - } - // laptop, judged from c1, goes; anchor waits. - p := rm.walk(t) - if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "could not be judged") { - t.Fatalf("the walk says of anchor %q", said) - } - if !slices.Contains(wentOut(p.Modules["mailu"].Recorded), "laptop") || rm.sentBuild(t, "laptop") != "c2" { - t.Fatalf("laptop, judged from c1, did not go: %+v", p.Modules["mailu"].Recorded) - } -} diff --git a/cmd/mesh-controller/recorded_push.go b/cmd/mesh-controller/recorded_push.go deleted file mode 100644 index c6ca5de3..00000000 --- a/cmd/mesh-controller/recorded_push.go +++ /dev/null @@ -1,153 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A push of recorded builds is no repair (novox/hq issue 301, ADR 0242, to-be 45 §7). -// -// **A recorded build moves only by a person's push** (ADR 0242): a module whose upgrade policy is -// `record` — the network path, the providers whose restart costs, mail — has its new build registered -// at its merge and sent by nothing but `push `. That push is the person's word the policy asks -// for: the mesh working as decided. The hand-act log counted it as a repair all the same, because a push -// by hand is the repair roll-out by default exists to end. On 2026-10-07 the operator approved, node by -// node, new builds of the resolver, the FortiClient adapter, the network managers and the packet filter; -// ten pushes later S15 wanted a healer for `split-dns`, `words` and `uplink-verbs`. -// -// **The push says what it was, from what it carried.** Never from a word the person gives — that is -// how a repair could pass for anything (issue 292). Before it is recorded, a push that names one machine -// reads what that machine was last sent against what the mesh holds. It is a **push of recorded builds** -// when: -// -// - what the machine was last sent is known; -// - its last report is no failure or refusal, so the push does not re-send to mend one; -// - it moves at least one module's build; and -// - every module it moves records rather than rolls out: a held new build of one the machine runs, or -// the first build of one newly assigned there (whose `assign` says "run `push ` to send it"). -// -// Anything else — a build that rolls out, a module taken off, nothing moved at all — is a push by hand -// as before, and counts. The act is written with `kind: recorded-builds` and what it carried; the table -// of verbs (handActVerbs) reads a push of that kind as a person's decision, and S15 never counts it. -// -// What is not compared: settings and grants. A push of a recorded build carries whatever else -// changed in the machine's declaration with it, as any push does. - -// recordedPush answers what a push to one machine carries, one "module from → to" each, when every -// build it moves is a recorded one held for a person's word; otherwise nil and why it is not. -// -// `modules` is the machine's set; `sent` and `known` what it was last sent (Inventory.SentBuilds); -// `current` what the mesh holds; `identical` whether two builds of a module put the same thing on a -// machine (moveFacts.identical); `failing` the machine's last report when it was a failure or a refusal. -func recordedPush(modules []string, sent map[string]string, known bool, current map[string]inventory.CurrentBuild, - identical func(module, a, b string) bool, failing string) ([]string, string) { - if !known { - return nil, "what the machine was last sent is not known" - } - if failing != "" { - return nil, "its last report was " + failing + ": the push sends again what failed" - } - in := map[string]bool{} - var carried []string - for _, m := range modules { - in[m] = true - // A module the mesh holds no build of, or holds without a source, has no build to move. - now, held := current[m] - if !held || now.Commit == "" { - continue - } - was, ran := sent[m] - if ran && identical(m, was, now.Commit) { - continue - } - if now.RollOut { - return nil, fmt.Sprintf("%s would move %sto %s, and it rolls out: its build is a plan's to send", m, - fromBuild(was, ran), buildName(now.Commit)) - } - carried = append(carried, fmt.Sprintf("%s %s→ %s", m, fromBuild(was, ran), buildName(now.Commit))) - } - for m := range sent { - if !in[m] { - return nil, m + " is taken off the machine" - } - } - if len(carried) == 0 { - return nil, "it moves no build: a send again" - } - sort.Strings(carried) - return carried, "" -} - -// recordedPushOf reads, for a push about to name one machine, whether it is a push of recorded builds: what -// it carries, -// or nil and why not. An error is that it could not be read; the push is then recorded as a push by hand, -// as every push was before, and says why. -func recordedPushOf(ctx context.Context, node string) ([]string, string, error) { - open, err := openStores(ctx) - if err != nil { - return nil, "", err - } - defer open.Close() - inv := open.inventory - plan, _, err := resolveNode(ctx, open, node) - if err != nil { - return nil, "", fmt.Errorf("its set cannot be worked out: %w", err) - } - modules := make([]string, 0, len(plan.Modules)) - for _, m := range plan.Modules { - modules = append(modules, m.Module) - } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil { - return nil, "", err - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, "", err - } - doing, said, err := inv.DoingOf(ctx, node) - if err != nil { - return nil, "", err - } - failing := "" - if said && (doing.Outcome == inventory.OutcomeFailed || doing.Outcome == inventory.OutcomeRefused) { - failing = doing.Outcome - } - carried, why := recordedPush(modules, sent, known, f.current, f.identical, failing) - return carried, why, nil -} - -// recordedBefore are the pushes recorded before a push said its kind, each a push of recorded builds by -// what it carried (novox/hq issue 301): the operator's word, machine by machine, for new builds of modules -// whose upgrade policy records — systemd-resolved, forticlient, networkmanager, systemd-networkd, -// nftables, mailu. The log did not keep what a push carried then, so the record names them; no push -// recorded since needs it, because every push now says its kind. S15 reads these as it reads a push of -// that kind, and the three `healer-wanted` they opened clear on the next tick. -var recordedBefore = map[string]string{ - "act-1791404241010309198-1": "systemd-resolved, first build on its machine (ADR 0247)", - "act-1791404587689907257-1": "systemd-resolved, first build; nftables, held build (catalogue #111)", - "act-1791404809925218992-1": "nftables, held build (catalogue #111)", - "act-1791404844504887009-1": "mailu and nftables, held builds (catalogue #106, #111)", - "act-1791408072745552019-1": "forticlient, held build (catalogue #114)", - "act-1791408101286318350-1": "forticlient, held build (catalogue #114)", - "act-1791409209593713522-1": "networkmanager, held build (catalogue #107)", - "act-1791409280125022264-1": "networkmanager, held build (catalogue #107)", - "act-1791409336586475835-1": "networkmanager, held build (catalogue #107)", - "act-1791409393494198352-1": "systemd-networkd, held build (catalogue #107)", -} - -// pushedRecorded is whether an act in the log is a push of recorded builds. -func pushedRecorded(a link.HandAct) bool { - if a.Verb != "push" { - return false - } - if a.Kind == link.KindRecordedBuilds { - return true - } - _, before := recordedBefore[a.ID] - return before && a.Kind == "" -} diff --git a/cmd/mesh-controller/recorded_push_test.go b/cmd/mesh-controller/recorded_push_test.go deleted file mode 100644 index 599cf43d..00000000 --- a/cmd/mesh-controller/recorded_push_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A push says whether it pushed recorded builds only, from what it carries** (novox/hq issue 301): -// every module it moves records rather than rolls out, the machine's last send is known and its last -// report no failure. Anything else is a push by hand, as before. -func TestAPushIsOfRecordedBuildsOnlyWhenEveryBuildItMovesWasHeldForAPersonsWord(t *testing.T) { - current := map[string]inventory.CurrentBuild{ - "resolver": {Commit: "r2222222"}, "nftables": {Commit: "n2222222"}, "mail": {Commit: "m2222222"}, - "web": {Commit: "w2222222", RollOut: true}, "proxy": {Commit: "p1111111", RollOut: true}, - } - same := func(_, a, b string) bool { return a == b } - for _, c := range []struct { - name string - modules []string - sent map[string]string - known bool - failing string - carried string // what it carried, joined; empty when it is not a push of recorded builds - why string - }{ - {name: "one held build", modules: []string{"resolver", "proxy"}, - sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true, - carried: "resolver from r1111111 → r2222222"}, - {name: "two held builds", modules: []string{"mail", "nftables", "proxy"}, - sent: map[string]string{"mail": "m1111111", "nftables": "n1111111", "proxy": "p1111111"}, known: true, - carried: "mail from m1111111 → m2222222; nftables from n1111111 → n2222222"}, - {name: "a recorded module's first build, newly assigned", modules: []string{"resolver", "proxy"}, - sent: map[string]string{"proxy": "p1111111"}, known: true, - carried: "resolver (never sent it) → r2222222"}, - {name: "a build that rolls out moves too", modules: []string{"resolver", "web"}, - sent: map[string]string{"resolver": "r1111111", "web": "w1111111"}, known: true, why: "rolls out"}, - {name: "nothing moves", modules: []string{"resolver", "proxy"}, - sent: map[string]string{"resolver": "r2222222", "proxy": "p1111111"}, known: true, why: "a send again"}, - {name: "a module taken off", modules: []string{"resolver"}, - sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true, why: "taken off"}, - {name: "what it was last sent is not known", modules: []string{"resolver"}, why: "not known"}, - {name: "its last report failed", modules: []string{"resolver"}, - sent: map[string]string{"resolver": "r1111111"}, known: true, failing: "failed", why: "failed"}, - } { - carried, why := recordedPush(c.modules, c.sent, c.known, current, same, c.failing) - if got := strings.Join(carried, "; "); got != c.carried { - t.Errorf("%s: carried %q, want %q (%s)", c.name, got, c.carried, why) - } - if c.why != "" && !strings.Contains(why, c.why) { - t.Errorf("%s: not of recorded builds because %q, want it to say %q", c.name, why, c.why) - } - } -} - -// **The live case, as the log holds it** (novox/hq issue 301). The ten pushes of 2026-10-07 recorded -// before a push said its kind pushed recorded builds only; the three `healer-wanted` they opened clear. -// The five pushes for the settings beside them, and any push that says no kind, still count; and a push -// recorded as one of recorded builds since, repeated, never does. -func TestThePushesOfRecordedBuildsOf20261007WantNoHealer(t *testing.T) { - now := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) - pushed := func(id, cause string, at time.Time) link.HandAct { - return link.HandAct{ID: id, At: at, By: "operator", Verb: "push", Args: []string{"laptop"}, - Why: "operator's yes", Cause: cause} - } - var acts []link.HandAct - at := now.Add(-12 * time.Hour) - causes := map[string]string{ - "act-1791404241010309198-1": "split-dns", "act-1791404587689907257-1": "split-dns", - "act-1791408072745552019-1": "split-dns", "act-1791408101286318350-1": "split-dns", - "act-1791404809925218992-1": "words", "act-1791404844504887009-1": "words", - "act-1791409209593713522-1": "uplink-verbs", "act-1791409280125022264-1": "uplink-verbs", - "act-1791409336586475835-1": "uplink-verbs", "act-1791409393494198352-1": "uplink-verbs", - } - if len(causes) != len(recordedBefore) { - t.Fatalf("the live case has %d acts and the record names %d", len(causes), len(recordedBefore)) - } - for id, cause := range causes { - if _, named := recordedBefore[id]; !named { - t.Fatalf("%s is not named a push of recorded builds", id) - } - acts = append(acts, pushed(id, cause, at)) - at = at.Add(time.Minute) - } - for i := range 2 { - acts = append(acts, pushed("act-settings-"+string(rune('a'+i)), "push", at)) - at = at.Add(time.Minute) - } - f := calm(now) - f.handActs = acts - got := watchHandActs(f) - if len(got) != 1 || got[0].ID != "hand-acts.push" { - t.Fatalf("S15 saw %+v; want only the settings pushes, counted as before", got) - } - - // A push that says it carried recorded builds, repeated, wants no healer; one recorded with the same word and no - // kind still counts, as does an act of another verb whose ID such a push once had. - ofRecorded := func(at time.Time) link.HandAct { - a := pushed("", "split-dns", at) - a.Kind, a.Carried = link.KindRecordedBuilds, []string{"resolver from r1111111 → r2222222"} - return a - } - f = calm(now) - f.handActs = []link.HandAct{ofRecorded(now.Add(-26 * time.Hour)), ofRecorded(now.Add(-time.Hour)), - ofRecorded(now.Add(-time.Minute))} - if got := watchHandActs(f); len(got) != 0 { - t.Errorf("pushes of recorded builds repeated asked for a healer: %+v", got) - } - f = calm(now) - f.handActs = []link.HandAct{pushed("a", "split-dns", now.Add(-time.Hour)), pushed("b", "split-dns", now.Add(-time.Minute))} - if got := watchHandActs(f); len(got) != 1 { - t.Errorf("pushes that said no kind were not counted: %+v", got) - } - other := actOf(now.Add(-time.Hour), "plans close", "split-dns") - other.ID = "act-1791404241010309198-1" - f = calm(now) - f.handActs = []link.HandAct{other, actOf(now.Add(-time.Minute), "plans close", "split-dns")} - if got := watchHandActs(f); len(got) != 1 { - t.Errorf("another verb under such a push's ID passed for one: %+v", got) - } -} diff --git a/cmd/mesh-controller/registry_verbs.go b/cmd/mesh-controller/registry_verbs.go index c89f695d..b0fe3b5a 100644 --- a/cmd/mesh-controller/registry_verbs.go +++ b/cmd/mesh-controller/registry_verbs.go @@ -6,6 +6,8 @@ import ( "errors" "flag" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" "os" "slices" "sort" @@ -413,18 +415,12 @@ func imagesCommand(ctx context.Context, args []string) error { return err } } - // And what it was last sent, as `plan --diff` compares against. - prev, err := open.inventory.SentSummary(ctx, node) + // And what its compositions name, which the node applies by its wants (ADR 0539 §2): read from its bucket on the bus. + sent, kept, err := keptImages(ctx, node) if err != nil { return err } - var sent []sentResource - if len(prev) > 0 { - if err := json.Unmarshal(prev, &sent); err != nil { - return fmt.Errorf("%s: what it was last sent is kept in a form this version does not read: %w", node, err) - } - } - answer, err := imagesOf(node, body, sent, len(prev) > 0) + answer, err := imagesOf(node, body, sent, kept) if err != nil { return err } @@ -437,7 +433,58 @@ func imagesCommand(ctx context.Context, args []string) error { fmt.Printf("%s (%s; %s)\n", d.Image, strings.Join(d.In, ", "), strings.Join(d.Resources, ", ")) } if !answer.SentKnown { - fmt.Printf("%s: which images it was last sent is not kept — the next push keeps it\n", node) + fmt.Printf("%s: no composition of it is kept on the bus yet, so which images it applies is not known\n", node) } return nil } + +// sentResource is a container among the resources of a node's compositions, as `images` reads it: the image the node applies. +type sentResource struct { + ID string `json:"id"` + Type string `json:"type"` + Image string `json:"image,omitempty"` +} + +// imagesOfCompositions are the containers of the compositions' resources, in the shape `images` compares with what would be +// declared; the bool says a composition was kept at all. +func imagesOfCompositions(values [][]byte) ([]sentResource, bool, error) { + var out []sentResource + for _, value := range values { + c, err := sdkdelivery.DecodeComposed(value) + if err != nil { + return nil, false, fmt.Errorf("a composition is not readable: %w", err) + } + for _, raw := range c.Resources { + var r struct { + ID string `json:"id"` + Type string `json:"type"` + Image string `json:"image"` + } + if err := json.Unmarshal(raw, &r); err != nil { + return nil, false, fmt.Errorf("a resource of a composition is not readable: %w", err) + } + if r.Type == "container" { + out = append(out, sentResource{ID: r.ID, Type: r.Type, Image: r.Image}) + } + } + } + return out, len(values) > 0, nil +} + +// keptImages reads a node's compositions from its bucket on the bus and answers their containers. +func keptImages(ctx context.Context, node string) ([]sentResource, bool, error) { + bus, err := aBus() + if err != nil { + return nil, false, fmt.Errorf("a node's compositions are kept on the bus, and this process has none: %w", err) + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return nil, false, err + } + values, err := keptCompositionsOf(ctx, js, node) + if err != nil { + return nil, false, err + } + return imagesOfCompositions(values) +} diff --git a/cmd/mesh-controller/registry_verbs_test.go b/cmd/mesh-controller/registry_verbs_test.go index ec883231..a58be51f 100644 --- a/cmd/mesh-controller/registry_verbs_test.go +++ b/cmd/mesh-controller/registry_verbs_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "net/http" "net/http/httptest" "slices" @@ -230,13 +231,34 @@ func TestImagesAreEveryContainerImageOfTheDeclarationAndOfWhatWasSent(t *testing } } -func TestTheSentSummaryKeepsAContainersImage(t *testing.T) { - summary, err := summarize([]byte(`{"resources":[{"id":"web.server","type":"container","image":"x@sha256:aa"}, - {"id":"web.state","type":"directory","path":"/srv"}]}`)) - if err != nil { - t.Fatal(err) +// **`images` reads what a node applies from its compositions** (ADR 0539 §2, issue 1945 B8): the containers among the +// resources of the values kept in its bucket, whatever module and fingerprint; none kept is not known. +func TestImagesAreReadFromTheCompositionsAMachineKeeps(t *testing.T) { + file := json.RawMessage(`{"id":"web.config","type":"file","module":"web","path":"/etc/web","content":"x"}`) + box := json.RawMessage(`{"id":"web.server","type":"container","name":"web","module":"web","image":"registry/web@sha256:` + strings.Repeat("a", 64) + `"}`) + other := json.RawMessage(`{"id":"db.server","type":"container","name":"db","module":"db","image":"registry/db@sha256:` + strings.Repeat("b", 64) + `"}`) + encode := func(module string, fp string, resources ...json.RawMessage) []byte { + v, err := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "anchor", Module: module, FP: fp, Resources: resources}) + if err != nil { + t.Fatal(err) + } + return v } - if summary[0].Image != "x@sha256:aa" || summary[1].Image != "" { - t.Errorf("summarized as %+v", summary) + web := encode("web", sdkdelivery.SourcePrefix2+strings.Repeat("1", 64), file, box) + db := encode("db", sdkdelivery.SourcePrefix2+strings.Repeat("2", 64), other) + + got, kept, err := imagesOfCompositions([][]byte{web, db}) + if err != nil || !kept || len(got) != 2 { + t.Fatalf("images %+v kept %v: %v", got, kept, err) + } + answer, err := imagesOf("anchor", nil, got, kept) + if err != nil || len(answer.Images) != 2 || !answer.SentKnown { + t.Fatalf("the answer %+v: %v", answer, err) + } + if none, kept, err := imagesOfCompositions(nil); err != nil || kept || len(none) != 0 { + t.Fatalf("no composition kept: %+v %v %v", none, kept, err) + } + if _, _, err := imagesOfCompositions([][]byte{[]byte("nope")}); err == nil { + t.Fatal("an unreadable composition was read as holding no image") } } diff --git a/cmd/mesh-controller/reissue_hearing_test.go b/cmd/mesh-controller/reissue_hearing_test.go deleted file mode 100644 index 9375c583..00000000 --- a/cmd/mesh-controller/reissue_hearing_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "encoding/json" - "testing" - - "github.com/novox/mesh-controller/internal/broker" -) - -// A membership issued again for what it hears changes nothing else of it (novox/hq ADR 0363 §6). -func TestAMembershipIssuedAgainChangesOnlyWhatItHears(t *testing.T) { - issued := []byte(`{"node":"ace","module":"reader","receives":{"x":[1]},"emits":"mesh.mod.reader.event.{event}"}`) - hears := []broker.HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}} - body, changed, err := withHearing(issued, hears) - if err != nil || !changed { - t.Fatalf("changed %v, %v", changed, err) - } - var got map[string]any - if err := json.Unmarshal(body, &got); err != nil { - t.Fatal(err) - } - if got["receives"] == nil || got["emits"] != "mesh.mod.reader.event.{event}" || got["hears"] == nil { - t.Fatalf("issued again as %s", body) - } - if _, again, _ := withHearing(body, hears); again { - t.Fatal("the same hearing issued again counts as a change") - } - gone, changed, err := withHearing(body, nil) - if err != nil || !changed { - t.Fatalf("a withdrawal: %v, %v", changed, err) - } - got = nil - if err := json.Unmarshal(gone, &got); err != nil { - t.Fatal(err) - } - if _, still := got["hears"]; still { - t.Fatalf("a withdrawn hearing is still issued: %s", gone) - } -} diff --git a/cmd/mesh-controller/related.go b/cmd/mesh-controller/related.go deleted file mode 100644 index 7c507af1..00000000 --- a/cmd/mesh-controller/related.go +++ /dev/null @@ -1,195 +0,0 @@ -package main - -import ( - "context" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// What relates two changes (novox/hq ADR 0304 decision 1, as ADR 0306 decision 3 reads it): two changes are -// related when they share a repository, a module, a delivery group's head, or an edge of the module graph -// between their modules. A module is every one the walk builds or sends: the modules its merges moved and the -// modules that depend on them, not only the ones the merges touched. Unrelated changes may walk beside each -// other (ADR 0304 decision 2, ADR 0282 decision 3); a closed batch is cut beside an open walk it is -// not related to, and a walk about to ask its first tier waits only for a started walk it is related to (issue 973, -// ADR 0407 decision 4). - -// footprint is what one change reaches: its repositories (lower case), its modules, and its delivery group -// heads (the head branch names of its merges). unknown says a merge of it was kept without the modules it -// moves, so what it reaches is not known: it is related to every change, never wrongly to none. -type footprint struct { - repos, modules, heads map[string]bool - unknown string -} - -func newFootprint() footprint { - return footprint{repos: map[string]bool{}, modules: map[string]bool{}, heads: map[string]bool{}} -} - -// footprintOfWalk is a walk's footprint: the repositories it carries and the modules it plans. A walk keeps no -// head of its merges; kept are the merges the record holds for it, whose announcements say their heads, and -// may be nil. -func footprintOfWalk(p inventory.Plan, kept []inventory.BatchedMerge) footprint { - f := newFootprint() - for _, c := range p.Carried() { - f.repos[strings.ToLower(c.Repository)] = true - } - for name := range p.Modules { - f.modules[name] = true - } - for _, m := range kept { - f.repos[strings.ToLower(m.Repository)] = true - if h := announcedOf(m).Head; h != "" { - f.heads[h] = true - } - } - return f -} - -// footprintOfBatch is an assembling batch's footprint: each merge's repository and head, and the modules its -// merges moved when heard, widened to their dependents (planOfMoves). A merge kept with no moves (one owed a -// later look, or kept for its packaging) is judged again at the cut and may move anything then: its batch is -// unknown, related to everything. -func footprintOfBatch(merges []inventory.BatchedMerge, edges []inventory.Edge) footprint { - f := newFootprint() - var moved []string - for _, m := range merges { - k := announcedOf(m) - f.repos[strings.ToLower(m.Repository)] = true - if k.Head != "" { - f.heads[k.Head] = true - } - if len(k.Moves) == 0 && f.unknown == "" { - f.unknown = "the modules a merge of " + m.Repository + " moves are not known until its batch is cut" - } - moved = append(moved, k.Moves...) - } - for _, name := range reachableFrom(moved, edges) { - f.modules[name] = true - } - return f -} - -// related says whether two changes are related, and the first reason in plain words; false and empty when -// they are not. -func related(a, b footprint, edges []inventory.Edge) (bool, string) { - if a.unknown != "" { - return true, a.unknown - } - if b.unknown != "" { - return true, b.unknown - } - if r := firstShared(a.repos, b.repos); r != "" { - return true, "both change " + r - } - if m := firstShared(a.modules, b.modules); m != "" { - return true, "both move " + m - } - if h := firstShared(a.heads, b.heads); h != "" { - return true, "both are members of group " + h - } - for _, e := range edges { - if a.modules[e.From] && b.modules[e.To] || b.modules[e.From] && a.modules[e.To] { - return true, e.From + " depends on " + e.To + " (" + e.Kind + ")" - } - } - return false, "" -} - -// firstShared is the first key, in order, both sets hold; empty when none. -func firstShared(a, b map[string]bool) string { - var both []string - for k := range a { - if b[k] { - both = append(both, k) - } - } - if len(both) == 0 { - return "" - } - sort.Strings(both) - return both[0] -} - -// openWalk is a walk open beside a change: its id, what it reaches, and whether it is on the controller's own path. -type openWalk struct { - id string - fp footprint - own bool -} - -// relatedOpen is the first open walk a change is related to, with the reason; empty when it is related to none -// and may walk beside them all (issue 973, ADR 0407 decision 4, ADR 0306 decision 3). -// -// **The controller's own path is related to everything** (decided for issue 973): a walk or batch that replaces -// the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside another, and -// nothing runs beside it, whatever the footprints say. -func relatedOpen(cand footprint, candOwn bool, open []openWalk, edges []inventory.Edge) (behind, why string) { - for _, w := range open { - if candOwn { - return w.id, "it is on the controller's own path" - } - if w.own { - return w.id, w.id + " is on the controller's own path" - } - if ok, why := related(cand, w.fp, edges); ok { - return w.id, why - } - } - return "", "" -} - -// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path -// (issue 973). -func ownWalk(p inventory.Plan, onPath map[string]string) bool { - if p.OwnPath() { - return true - } - for name := range p.Modules { - if _, own := onPath[name]; own { - return true - } - } - return false -} - -// startedWalks are the open walks started beside the walk id, made at created: a merge's walk past its wait, not -// the backlog's, with a tier asked, or on its way (let go, or waiting for nobody) before this one. -func startedWalks(plans []inventory.Plan, id string, created time.Time) []inventory.Plan { - var out []inventory.Plan - for _, q := range plans { - if q.ID == id || q.Release != nil || q.Waiting() { - continue - } - if q.Tier > 0 || askedAny(q) || q.Created.Before(created) { - out = append(out, q) - } - } - return out -} - -// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the -// merges the record holds for it. -func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan, onPath map[string]string) (openWalk, error) { - kept, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return openWalk{}, err - } - return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p, onPath)}, nil -} - -// openWalksOf are openWalkOf for each of plans. -func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan, onPath map[string]string) ([]openWalk, error) { - var out []openWalk - for _, p := range plans { - w, err := openWalkOf(ctx, inv, p, onPath) - if err != nil { - return nil, err - } - out = append(out, w) - } - return out, nil -} diff --git a/cmd/mesh-controller/related_integration_test.go b/cmd/mesh-controller/related_integration_test.go deleted file mode 100644 index b29fb967..00000000 --- a/cmd/mesh-controller/related_integration_test.go +++ /dev/null @@ -1,52 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) { - edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}} - walk := func(id, repo string, own bool, modules ...string) openWalk { - return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own} - } - one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget") - cand := func(repo, head string, moves ...string) footprint { - return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges) - } - cases := []struct { - name string - fp footprint - own bool - open []openWalk - behind string - why string - }{ - {"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""}, - {"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""}, - {"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"}, - {"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two}, - "plan-2", "both move nzbget"}, - {"a graph edge", cand("novox/x", "fix/a", "nzbget"), - false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3", - "nzbget depends on base-image (stands-on)"}, - {"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1", - "the modules a merge of novox/x moves are not known until its batch is cut"}, - {"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1", - "it is on the controller's own path"}, - {"an open walk on the own path", cand("novox/x", "fix/a", "app"), false, - []openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4", - "plan-4 is on the controller's own path"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - behind, why := relatedOpen(c.fp, c.own, c.open, edges) - if behind != c.behind || why != c.why { - t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why) - } - }) - } -} diff --git a/cmd/mesh-controller/related_test.go b/cmd/mesh-controller/related_test.go deleted file mode 100644 index d67c0cb9..00000000 --- a/cmd/mesh-controller/related_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// heardMerge is a kept merge of a repository from a head branch that moved modules. -func heardMerge(repository, head string, moves ...string) inventory.BatchedMerge { - m := inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: "c-" + repository} - m.Event, _ = json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Head: head}, Moves: moves}) - return m -} - -// walkOf is a walk carrying one repository's commit and planning modules. -func walkOf(repository string, modules ...string) inventory.Plan { - p := inventory.Plan{Repository: repository, Commit: "w-" + repository, Modules: map[string]*inventory.PlanModule{}} - for _, name := range modules { - p.Modules[name] = &inventory.PlanModule{} - } - return p -} - -// TestRelatedChanges: two changes are related by a repository, a module, a group head or a graph edge between -// their modules, and by a module reached only through dependents (novox/hq ADR 0304 decision 1, ADR 0306 -// decision 3); otherwise they are not. -func TestRelatedChanges(t *testing.T) { - edges := []inventory.Edge{ - {From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}, - {From: "app", To: "build-machine", Kind: inventory.EdgeBuiltBy}, - } - cases := []struct { - name string - a, b footprint - why string - }{ - {"repository, case aside", - footprintOfWalk(walkOf("novox/Mesh-Catalog", "notes"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("NOVOX/mesh-catalog", "fix/a", "app")}, nil), - "both change novox/mesh-catalog"}, - {"module", - footprintOfWalk(walkOf("novox/one", "nzbget"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "nzbget")}, nil), - "both move nzbget"}, - {"edge, walk depends on batch", - footprintOfWalk(walkOf("novox/one", "nzbget"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, nil), - "nzbget depends on base-image (stands-on)"}, - {"edge, batch depends on walk", - footprintOfWalk(walkOf("novox/one", "build-machine"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "app")}, nil), - "app depends on build-machine (built-by)"}, - {"group head", - footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/x", "app")}, nil), - "both are members of group feat/x"}, - {"module reached through dependents", - footprintOfWalk(walkOf("novox/one", "nzbget"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, edges), - "both move nzbget"}, - {"unrelated", - footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/y", "app")}, edges), - ""}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - for _, pair := range [][2]footprint{{c.a, c.b}, {c.b, c.a}} { - is, why := related(pair[0], pair[1], edges) - if is != (c.why != "") || why != c.why { - t.Errorf("related = %v, %q; want %q", is, why, c.why) - } - } - }) - } -} - -// TestFootprintOfBatchWidensToDependents: a batch's modules are what its merges moved and every module that -// depends on them, along the edges that widen a walk. -func TestFootprintOfBatchWidensToDependents(t *testing.T) { - edges := []inventory.Edge{ - {From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}, - {From: "base-image", To: "build-machine", Kind: inventory.EdgeBuiltBy}, - } - f := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "", "base-image")}, edges) - if !f.modules["base-image"] || !f.modules["nzbget"] || f.modules["build-machine"] || len(f.heads) != 0 { - t.Errorf("footprint = %+v; want base-image and nzbget, no head", f) - } -} - -// A batch holding a merge kept with no moves (one owed a later look, judged again at the cut) is related to -// every change, in both orders: what it will move is not known, and an unknown change is never unrelated -// (issue 525 review). -func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) { - unknown := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app"), heardMerge("novox/two", "fix/b")}, nil) - other := footprintOfWalk(walkOf("novox/three", "nzbget"), nil) - for _, pair := range [][2]footprint{{unknown, other}, {other, unknown}} { - if ok, why := related(pair[0], pair[1], nil); !ok || !strings.Contains(why, "not known") { - t.Fatalf("an unknown batch read as %v (%q)", ok, why) - } - } - known := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app")}, nil) - if ok, why := related(known, other, nil); ok { - t.Fatalf("a known, unrelated batch read as related: %q", why) - } -} - -// TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo: a change that shares nothing with any open walk is -// held behind none (issue 973, ADR 0407 decision 4); one that shares a repository, a module, a graph edge or a -// group head is held behind the first walk it shares it with, with the reason; one whose moves are not known -// is held behind the first; a walk or change on the controller's own path is related to every other. - -// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973). -func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) { - if ownWalk(walkOf("novox/one", "notes"), ownPathOf(nil)) { - t.Error("an ordinary walk read as on the controller's own path") - } - flagged := walkOf("novox/one", "notes") - flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}} - if !ownWalk(flagged, ownPathOf(nil)) { - t.Error("a walk whose batch is own-path read as ordinary") - } - var module string - for m := range onTheControllersPath { - module = m - } - if !ownWalk(walkOf("novox/one", "notes", module), ownPathOf(nil)) { - t.Errorf("a walk planning %s read as ordinary", module) - } -} - -// A walk about to ask its first tier waits only for a started walk it is related to (issue 973): started is a -// tier asked, or a walk made before it; a waiting walk, the backlog's and itself are not. -func TestWalksStartedBesideAreTheOpenOnesBeforeIt(t *testing.T) { - at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - mk := func(id string, created time.Time, tier int) inventory.Plan { - p := walkOf("novox/"+id, id) - p.ID, p.Created, p.Tier, p.State = id, created, tier, inventory.PlanBuilding - return p - } - older, newer, ticking := mk("older", at, 0), mk("newer", at.Add(time.Minute), 0), mk("ticking", at.Add(2*time.Minute), 1) - backlog := mk("backlog", at, 1) - backlog.Release = &inventory.PlanRelease{} - got := startedWalks([]inventory.Plan{older, newer, ticking, backlog, mk("me", at.Add(time.Minute), 0)}, "me", at.Add(time.Minute)) - var ids []string - for _, p := range got { - ids = append(ids, p.ID) - } - if strings.Join(ids, ",") != "older,ticking" { - t.Errorf("started beside me = %v; want older, ticking", ids) - } -} - -// A retry is refused beside an open walk it is related to and allowed beside one it shares nothing with -// (issue 973); a retry on the controller's own path is refused beside any open walk. -func TestARetryWaitsOnlyForAnOpenWalkItIsRelatedTo(t *testing.T) { - failed := walkOf("novox/one", "notes") - failed.ID, failed.State = "plan-1", inventory.PlanFailed - open := func(id, repo string, modules ...string) inventory.Plan { - p := walkOf(repo, modules...) - p.ID, p.State = id, inventory.PlanBuilding - return p - } - if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err != nil { - t.Errorf("refused beside an unrelated walk: %v", err) - } - err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")}, ownPathOf(nil)) - if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") { - t.Errorf("a retry beside a walk moving the same module: %v", err) - } - failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}} - if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err == nil { - t.Error("an own-path retry went beside an open walk") - } -} - -// Plans another controller is working are "not now", not a failure: the merge is held and handed over again -// (issue 921). -func TestPlansHeldByAnotherControllerAreNotNow(t *testing.T) { - if err := notNow(fmt.Errorf("x: %w", inventory.ErrPlansBusy)); !errors.Is(err, link.ErrTryAgain) { - t.Errorf("busy plans read as %v; want ErrTryAgain", err) - } - if err := notNow(errors.New("no such module")); errors.Is(err, link.ErrTryAgain) { - t.Errorf("another failure read as ErrTryAgain: %v", err) - } -} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go deleted file mode 100644 index 348468c8..00000000 --- a/cmd/mesh-controller/release.go +++ /dev/null @@ -1,1689 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// No build reaches a machine without a gate (novox/hq ADR 0236). -// -// **A send carries the machine's whole declaration.** A plan sending one module to its first machine -// sends every other module whose build moved there too; a cascade, a healer's resend and a whole-mesh -// push do the same. Under the old default (`record`) builds were registered and sent nowhere, so on the -// day the default became `roll` every machine was behind on dozens of builds no gate had seen — and the -// next send of anything would have restarted all of them at once, on every machine. -// -// So **a build moves on a machine only through a send that judges it there**, or a person's: -// -// - a gated send — a plan's first machine, a release plan's machine — carries every move waiting on that -// machine, and its gate judges each of them; a pass is each build's verdict, a failure puts back what -// failed; -// - a module a send exists for may move anywhere it is sent: a policy of *together*, a rollback, a build -// whose gate already passed; -// - a send naming a machine by a person (`push `, the bus step) carries what it carries; -// - every other send — a plan's "rest", a cascade, a healer's, the bus's user list carried — is refused, -// or leaves the machine, while a move there waits for a gate. A rebuild that made the same artifacts -// from the same manifest is no move. -// -// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no -// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest -// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own: -// a person releases it again (`upgrade release-backlog --why`), after the condition says why. - -// sendScope is what a send may carry that no gate has seen. -type sendScope struct { - // judged are the machines whose every move this send's gate judges. - judged map[string]bool - // modules are those a send exists for, which may move wherever it goes. - modules map[string]bool - // person is a person's act: it carries what it carries. - person bool - // puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build - // older than the one its machine runs (novox/hq issue 201). - puttingBack bool -} - -type sendScopeKey struct{} - -func withScope(ctx context.Context, s sendScope) context.Context { - return context.WithValue(ctx, sendScopeKey{}, s) -} - -func scopeOf(ctx context.Context) sendScope { - s, _ := ctx.Value(sendScopeKey{}).(sendScope) - return s -} - -// errUngated is a send refused because it would carry a build no gate has seen. -var errUngated = errors.New("a build waits there for its gate") - -// errWalkedElsewhere is a gated send refused because a plan that has started walks a move there. -var errWalkedElsewhere = errors.New("a plan already walking a build there sends it") - -// moveFacts is what tells a move from a rebuild, and a gated build from one no gate has seen. -type moveFacts struct { - current map[string]inventory.CurrentBuild - fps map[string]map[string]string - // srcs is, per module, per commit, its build's source fingerprint (novox/hq issue 280). - srcs map[string]map[string]string - passed map[string]map[string]bool - plans []inventory.Plan - // self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353). - self string - // judge judges a move under a module's own `record` (novox/hq ADR 0377): nil when the facts were not read, - // and every such move then waits. - judge func(module, was, now string) recordVerdict -} - -type walkingKey struct{} - -// withWalking says a release plan is the one sending: its own gates never hold its own sends back. -func withWalking(ctx context.Context, id string) context.Context { - return context.WithValue(ctx, walkingKey{}, id) -} - -func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) { - var f moveFacts - var err error - if f.current, err = inv.CurrentBuilds(ctx); err != nil { - return f, err - } - if f.fps, err = inv.Fingerprints(ctx); err != nil { - return f, err - } - if f.srcs, err = inv.SourceFingerprints(ctx); err != nil { - return f, err - } - if f.passed, err = inv.PassedCommits(ctx); err != nil { - return f, err - } - f.plans, err = inv.OpenPlans(ctx) - f.self, _ = ctx.Value(walkingKey{}).(string) - f.judge = recordJudge(ctx, inv) - return f, err -} - -// identical is whether two builds of a module put the same thing on a machine: the same commit, -// builds made from the same source (novox/hq issue 280) — the module's tree, the contexts it read, its -// bases and toolchains, whatever digests an image rebuild made of them — or builds that made the same -// artifacts from the same manifest. -func (f moveFacts) identical(module, a, b string) bool { - if a == b || sameCommit(a, b) { - return true - } - if sa := f.srcs[module][a]; sa != "" && sa == f.srcs[module][b] { - return true - } - fa := f.fps[module][a] - return fa != "" && fa == f.fps[module][b] -} - -// gated is whether a build of a module from this commit — or one identical to it — passed a gate. -func (f moveFacts) gated(module, commit string) bool { - for c := range f.passed[module] { - if f.identical(module, c, commit) { - return true - } - } - return false -} - -// unchangedOnEvery is whether a plan's build of a module was made from the source of the build every -// machine running it was last sent (novox/hq issue 280): no move on any of them. False when no machine -// runs it, when what one was sent is not known, or when the build stands for itself. -func unchangedOnEvery(ctx context.Context, inv *inventory.Inventory, module, build string, running []string) (bool, error) { - if build == "" || len(running) == 0 { - return false, nil - } - same, err := inv.SameSourceCommits(ctx, module, build) - if err != nil || len(same) == 0 { - return false, err - } - for _, n := range running { - sent, known, err := inv.SentBuilds(ctx, n) - if err != nil { - return false, err - } - was, carried := sent[module] - if !known || !carried || !inRun(same, was) { - return false, nil - } - } - return true, nil -} - -// inRun is whether a commit is one of a run's, however either is abbreviated. -func inRun(run map[string]bool, commit string) bool { - for c := range run { - if sameCommit(c, commit) { - return true - } - } - return false -} - -// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out -// (a recorded one is a person's push, unless its move under its own record recreates nothing and moves no data — -// novox/hq ADR 0377), that the machine was sent before, moving to a build not identical -// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names -// none: the cascade holds it whole (ADR 0221). -// -// With all, a move to a build that passed a gate elsewhere counts too: what a gated send carries. -func (f moveFacts) moves(node string, modules []string, sent map[string]string, known, all bool) []inventory.CarriedMove { - if !known { - return nil - } - var out []inventory.CarriedMove - for _, m := range modules { - now := f.current[m] - was, carried := sent[m] - if !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) || !f.rolls(m, was) { - continue - } - out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - -// walkedBy is the open plan that has started walking a module's build — sent it to a first machine, not -// yet passed — and whose walk this move would cross; empty when none does. A move of the same build to a -// machine that plan already sent it is not a crossing: the build is there. A move of **another** build of -// the module to that machine is (novox/hq issue 352): on 2026-10-09 a merge's plan sent the control node a -// newer controller while a release's gate was judging the controller there, the release's gate read the -// machine's report against the newer send, failed three builds and put them back under the new plan's -// feet. A release keeps no module records: its open gate's carried moves are its walk. -func (f moveFacts) walkedBy(module, node, to string) string { - for _, p := range f.plans { - if !p.Open() || (p.Release != nil && p.ID == f.self) { - continue - } - if p.Release != nil { - // The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353). - gates := []*inventory.PlanGate{p.Release.Gate} - for _, g := range p.Release.Gates { - gates = append(gates, g) - } - for _, g := range gates { - // A send kept and not yet made walks nothing yet (ADR 0353 §8). - if g == nil || g.Verdict != "" || g.Silent || g.Pending { - continue - } - for _, c := range g.Carried { - if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) { - return p.ID - } - } - } - continue - } - s, holds := p.Modules[module] - if !holds || s == nil || s.FirstAt == nil || s.SentAt != nil { - continue - } - if slices.Contains(s.First, node) && sameCommit(s.Commit, to) { - continue - } - if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed { - continue - } - return p.ID - } - return "" -} - -// machineMoves is the moves no gate has seen on one machine, read from what it would be sent now. -var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - plan, _, err := resolveNode(ctx, open, node) - if err != nil { - return nil, nil // it cannot be worked out: the send says why - } - modules := make([]string, 0, len(plan.Modules)) - for _, m := range plan.Modules { - modules = append(modules, m.Module) - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil -} - -// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does -// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the -// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that -// call this), so a move it could not keep is still counted. -func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) { - if len(moves) == 0 { - return moves, nil - } - keeping := ctx - if _, on := keptExcept(ctx); !on { - keeping = keepingRecorded(ctx) - } - keeps, err := recordedKept(keeping, open, node) - if err != nil || len(keeps) == 0 { - return moves, err - } - var out []inventory.CarriedMove - for _, mv := range moves { - if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { - continue - } - out = append(out, mv) - } - return out, nil -} - -// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the -// machines named, and why; the machine holding the bus, added only for its user list, is left instead. -func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) { - scope := scopeOf(ctx) - if scope.person { - return names, nil - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return nil, err - } - var kept, refused []string - for _, n := range names { - moves, err := machineMoves(ctx, open, f, n, false) - if err != nil { - return nil, err - } - if moves, err = withoutKept(ctx, open, n, moves); err != nil { - return nil, err - } - var waiting []string - for _, mv := range moves { - if !scope.judged[n] && !scope.modules[mv.Module] { - waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) - } - } - switch { - case len(waiting) == 0: - kept = append(kept, n) - case n == addedHolder: - fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s wait there for a gate "+ - "— the bus may refuse what was newly granted until it is\n", n, strings.Join(waiting, ", ")) - default: - refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", "))) - } - } - if len(refused) > 0 { - return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+ - "0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) - } - return kept, nil -} - -// gatedSend sends one machine everything waiting there, under a gate that judges it all: what moved is -// answered, with the build each moved to, for the gate to judge and to put back. Refused while a plan that -// has started walks one of those builds elsewhere: that plan sends it here once its gate passed. -// -// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in -// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per -// module was the same declaration sent again and again, each one setting aside the one before. -// -// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490): -// answered for the gate to keep, nil when there was none. -func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) { - moves, err := gatedMoves(ctx, open, node, owns) - if err != nil || len(moves) == 0 { - return nil, nil, nil, err - } - sayRecreations(ctx, open, moves) - sent, grants, err := sendJudged(ctx, open, node) - if err != nil { - return nil, nil, nil, err - } - return moves, sent, grants, nil -} - -// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing: -// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way -// while a plan that has started walks one of those builds elsewhere. -func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) { - inv := open.inventory - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - moves, err := machineMoves(ctx, open, f, node, true) - if err != nil { - return nil, err - } - // A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module - // kept over an older record judged as moving back to it would read the gate as superseded by its own send. - if moves, err = withoutKept(ctx, open, node, moves); err != nil { - return nil, err - } - own := func(module string) bool { - return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) - } - for _, mv := range moves { - if own(mv.Module) { - continue - } - if id := f.walkedBy(mv.Module, node, mv.To); id != "" { - return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, - mv.Module, short(mv.To), node, id) - } - } - // **And the plan's own modules wait too** (novox/hq issue 352): a walk of the same module by another - // plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate. - for _, o := range owns { - if id := f.walkedBy(o.Module, node, o.To); id != "" { - return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, - o.Module, short(o.To), node, id) - } - } - for _, o := range owns { - i := slices.IndexFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == o.Module }) - switch { - case i < 0: - moves = append(moves, o) - case moves[i].Build == "": - moves[i].Build = o.Build - } - } - if len(moves) == 0 { - return nil, nil - } - for i := range moves { - if moves[i].Build == "" { - if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil { - return nil, err - } - } - } - sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module }) - return moves, nil -} - -// passCarried keeps a pass as the verdict of every build the gate judged beside its own module. -func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { - seen := map[string]bool{} - for _, c := range g.Carried { - // A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3). - if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) { - continue - } - seen[c.Build] = true - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, - Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: whyFor(g, c.Module), - Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { - fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err) - } - } -} - -// failCarried puts back every build the gate carried that it found wanting, on the machines that were -// sent it, once each. -func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { - var notes []string - if p.Note != "" { - notes = append(notes, p.Note) - } - done := map[string]bool{except: true} - for _, m := range g.Returned { - done[m] = true // put back at once when it broke - } - // **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the - // gate asks while another module of the send failed, it is neither put back nor left unjudged — a build - // left without a verdict is one more move every other walk would wait for. - if kept := keepPassing(ctx, open, p, g, except); len(kept) > 0 { - notes = append(notes, "passed on their own and kept: "+strings.Join(kept, ", ")) - for _, m := range kept { - done[m] = true - } - } - for _, c := range g.Carried { - if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) { - continue - } - done[c.Module] = true - machines, err := sentTheBuild(ctx, open, c.Module, c.To) - if err != nil || len(machines) == 0 { - machines = []string{c.Node} - } - state := &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To} - // A module of the plan's tier sent in the same send (issue 281) keeps its own record of it. - if s := p.Modules[c.Module]; s != nil && except != "" && s.GatedBy == except && s.Build == c.Build { - state = s - } - p.Note = "" - gateFailed(ctx, open, p, c.Module, state, machines, g.Why) - notes = append(notes, p.Note) - } - p.State = inventory.PlanFailed - p.Note = strings.Join(notes, "; ") -} - -// keepPassing keeps a pass as the verdict of every build the gate carried that passed on its own when the -// send failed (ADR 0254), except the plan's own module; answers the modules kept. -func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) []string { - var kept []string - seen := map[string]bool{} - for _, c := range g.Carried { - if c.Module == except || c.Build == "" || seen[c.Build] || !slices.Contains(g.Passing, c.Module) { - continue - } - seen[c.Build] = true - why := passedAloneWhy(g, c.Module) - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, - Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why, - Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { - fmt.Printf("%s: %s passed its gate on %s on its own, and the verdict could not be kept: %v\n", p.ID, c.Module, - c.Node, err) - continue - } - if !slices.Contains(kept, c.Module) { - kept = append(kept, c.Module) - } - } - return kept -} - -// putBackBroken puts back, at once, every module a gate found broken and has not put back yet (issue 318 -// review): a witness that reverted a core component, or a machine that refused or failed its send, is not -// left registered at the build that broke for as long as the modules beside it take to be judged — a resend -// meanwhile would declare it again. The send goes on judging the others to their own verdicts; its own -// verdict is still failed. lead and leadState are the module a plan's gate is kept on, when it is a plan's. -// Answers whether anything was put back. -func putBackBroken(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, lead string, - leadState *inventory.PlanModule) bool { - var todo []string - for _, m := range g.Broken { - if !slices.Contains(g.Returned, m) { - todo = append(todo, m) - } - } - if len(todo) == 0 || g.Verdict != "" { - return false - } - stateWas, noteWas := p.State, p.Note - batched, back := batchingRollbacks(ctx) - var said []string - for _, m := range todo { - machines := g.Machines - var state *inventory.PlanModule - if m == lead && leadState != nil { - judged := *leadState - judged.Gate = nil // its own record of the failure; the send's gate goes on judging - state = &judged - } else { - i := slices.IndexFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == m }) - if i < 0 { - continue - } - c := g.Carried[i] - state = &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To} - if sent, err := sentTheBuild(ctx, open, m, c.To); err == nil && len(sent) > 0 { - machines = sent - } else { - machines = []string{c.Node} - } - } - // Counted as put back only once there is something to put back (a carried move or the plan's own). - g.Returned = append(g.Returned, m) - p.Note = "" - gateFailed(batched, open, p, m, state, machines, g.BrokenWhy) - if m == lead && leadState != nil { - leadState.Why = "put back at once, its send still judged: " + g.BrokenWhy - } - said = append(said, m) - } - sendRollbacks(ctx, open, p, back) - p.State = stateWas - p.Note = noteWas - if len(said) > 0 { - p.Note = fmt.Sprintf("put back at once: %s (%s); the rest of the send judged to their own verdicts", - strings.Join(said, ", "), g.BrokenWhy) - fmt.Printf("%s: %s\n", p.ID, p.Note) - } - return true -} - -// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait. -func passedAloneWhy(g *inventory.PlanGate, module string) string { - why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why) - if w := g.Waits[module]; w != "" { - why += waitsPrefix + w - } - return why -} - -// sentTheBuild is every machine running a module that was last sent this build of it. -func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) { - running, err := open.inventory.Running(ctx, module) - if err != nil { - return nil, err - } - var out []string - for _, n := range running { - sent, known, err := open.inventory.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - if known && sameCommit(sent[module], commit) { - out = append(out, n) - } - } - return out, nil -} - -// releaseRepository is what a release plan says it is for, where a merge's says its repository. -const releaseRepository = "the builds waiting for a gate" - -// releaseHeard is the machines a release plan may send: heard within their heartbeat's bound. A machine -// away is left, not judged against a bound it cannot meet. A variable so a test says who is heard. -var releaseHeard = func(ctx context.Context, open *stores) (map[string]bool, error) { - if d := doctorFrom; d != nil && d.watchdogs != nil { - return heardMachines(d), nil - } - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return nil, err - } - out := map[string]bool{} - for _, r := range reports { - if r.At != nil && time.Since(*r.At) < 15*time.Minute { - out[r.Node] = true - } - } - return out, nil -} - -// backlogNow is what the newest look found waiting, for `upgrade backlog` and the gate's probe. -var backlogNow struct { - held string - waiting map[string][]inventory.CarriedMove -} - -// waitingMoves is every machine's moves no gate has seen and no started plan walks. -func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inventory.CarriedMove, error) { - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return nil, err - } - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return nil, err - } - out := map[string][]inventory.CarriedMove{} - for _, n := range nodes { - moves, err := machineMoves(ctx, open, f, n.Name, all) - if err != nil { - return nil, err - } - for _, mv := range moves { - if f.walkedBy(mv.Module, n.Name, mv.To) == "" { - out[n.Name] = append(out[n.Name], mv) - } - } - } - return out, nil -} - -// releaseBacklog opens a release plan when builds wait for a gate and none is open; not after a release -// plan failed, until a person releases one (by). Called with the plans held. -func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Plan, error) { - inv := open.inventory - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - for _, p := range plans { - if p.Release != nil { - if by != "" && p.Release.Settings != nil { - return nil, fmt.Errorf("%s is sending %s; what waits is released once it ends — `plans %s` says where it is", - p.ID, p.Repository, p.ID) - } - if by != "" { - return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID) - } - return nil, nil - } - } - waiting, err := waitingMoves(ctx, open, false) - if err != nil { - return nil, err - } - backlogNow.waiting, backlogNow.held = waiting, "" - if len(waiting) == 0 { - return nil, nil - } - // Opened by what no gate has seen; it walks every machine where anything of the release waits — a - // build that passed on the first machine still goes to the next one by this plan, judged there too. - if waiting, err = waitingMoves(ctx, open, true); err != nil { - return nil, err - } - if by == "" { - recent, err := inv.RecentPlans(ctx, 50) - if err != nil { - return nil, err - } - for _, p := range recent { - // A settings walk that failed holds nothing back (novox/hq ADR 0344 §6): its failure is its module's - // settings on one machine, said on its own, not a build no gate passed. - if p.Release == nil || p.Release.Settings != nil { - continue - } - if p.State == inventory.PlanFailed { - backlogNow.held = fmt.Sprintf("%s failed (%s); what waits is released again by a person", p.ID, p.Note) - return nil, nil - } - break - } - } - heard, err := releaseHeard(ctx, open) - if err != nil { - return nil, err - } - controllers, err := inv.Running(ctx, catalogue.ControllerSeatName) - if err != nil { - return nil, err - } - var order, last []string - modules := map[string]bool{} - for node, moves := range waiting { - if !heard[node] { - continue - } - for _, mv := range moves { - modules[mv.Module] = true - } - if slices.Contains(controllers, node) { - last = append(last, node) - } else { - order = append(order, node) - } - } - if len(order)+len(last) == 0 { - return nil, nil - } - sort.Strings(order) - sort.Strings(last) - order = append(order, last...) - names := make([]string, 0, len(modules)) - for m := range modules { - names = append(names, m) - } - sort.Strings(names) - now := time.Now().UTC() - p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository, - Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{}, - Release: &inventory.PlanRelease{Order: order, By: by}, - Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names), - strings.Join(order, ", "))} - if err := inv.SavePlan(ctx, &p); err != nil { - return nil, err - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return &p, nil -} - -// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a -// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR -// 0353 §1), and judged; the plan done when every machine is. -// -// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with -// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it -// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged. -func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { - r := p.Release - if r.Settings != nil { - return advanceSettingsWalk(ctx, open, p) - } - ctx = withWalking(ctx, p.ID) - now := time.Now().UTC() - if r.Rest { - return advanceRest(ctx, open, p, now) - } - if r.Gate == nil && !gateFirstMachine { - // **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step. - r.Rest = true - return advanceRest(ctx, open, p, now) - } - if r.Gate == nil { - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, err - } - for r.Next < len(r.Order) { - node := r.Order[r.Next] - if !heard[node] { - r.Skipped = append(r.Skipped, node) - r.Next++ - continue - } - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - note := fmt.Sprintf("waiting before %s: %v", node, err) - changed := p.Note != note - p.Note = note - return changed, nil - } - if err != nil { - return false, fmt.Errorf("reading what waits on %s: %w", node, err) - } - if len(moves) == 0 { - r.Done = append(r.Done, node) - r.Next++ - continue - } - r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true} - p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves)) - return true, nil - } - p.State, p.Tier = inventory.PlanDone, len(p.Tiers) - p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) - if len(r.Skipped) > 0 { - p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - return true, nil - } - g := r.Gate - if g.Pending { - node := firstOf(g.Machines) - out, err := sendKept(ctx, open, p, node, g, now) - if err != nil || out == keptWaiting { - return out == keptWaiting, err - } - if out == keptNothing { - r.Gate = nil - r.Done = append(r.Done, node) - r.Next++ - } - return true, nil - } - verdict, err := judgeMoves(ctx, open, g, nil, now) - if err != nil { - return false, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, g, "", nil) { - return true, nil - } - note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g)) - changed := p.Note != note - p.Note = note - return changed, nil - case inventory.GatePassed: - passCarried(ctx, open, p, g, "") - r.Done = append(r.Done, firstOf(g.Machines)) - r.Next++ - r.Gate = nil - // **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own - // gate, in place of one machine at a time (ADR 0236 §22). - r.Rest = true - p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines)) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - case inventory.GateSuperseded: - // Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what - // was carried, nothing put back, and this release ends; the builds still waiting are released again - // by the next pass, judged afresh. - p.State = inventory.PlanSuperseded - p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; what still waits is released again", - strings.Join(g.Machines, ", "), g.Why) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - p.Note = "" - batched, back := batchingRollbacks(ctx) - failCarried(batched, open, p, g, "") - sendRollbacks(ctx, open, p, back) - p.Note = fmt.Sprintf("failed its gate on %s: %s — %s", strings.Join(g.Machines, ", "), g.Why, p.Note) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// What sendKept did with a kept send. -const ( - keptSent = iota // sent now - keptAlready // the machine was already sent it: judged from that send, nothing sent - keptNothing // nothing waits there any more: no gate - keptWaiting // another walk judges a build it would carry: it waits -) - -// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send -// already carries what the gate is for, made before a controller was replaced and before it was kept: then -// nothing is sent, and the gate judges from that send (§6). -func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate, - now time.Time) (int, error) { - inv := open.inventory - moves, sent, grants, err := gatedSend(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - p.Note = fmt.Sprintf("waiting before %s: %v", node, err) - return keptWaiting, nil - } - if err != nil { - return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err) - } - if len(moves) > 0 { - g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false - g.Sent = sentNow(ctx, inv, sent) - p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - if said := recreationsSaid(moves); said != "" { - p.Note += "; " + said - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return keptSent, nil - } - if !alreadySent(ctx, inv, node, g.Carried) { - return keptNothing, nil - } - since := now - if s, found, err := inv.SentTo(ctx, node); err == nil && found { - if !s.At.IsZero() { - since = s.At - } - g.Sent = map[string]inventory.SentDeclaration{node: s} - } - g.Since, g.Pending = &since, false - p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again", - node, len(g.Carried), since.Local().Format("15:04:05")) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return keptAlready, nil -} - -// alreadySent says a machine's last send carries every build of the moves: a send made and not kept. -func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool { - if len(moves) == 0 { - return false - } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known { - return false - } - for _, mv := range moves { - if !sameCommit(sent[mv.Module], mv.To) { - return false - } - } - return true -} - -// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every -// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their -// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent. -func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) { - r := p.Release - // The sends kept for this step, in the plan's order: the control node, last in it, last (§1). - var pending []string - for _, n := range r.Order { - if g := r.Gates[n]; g != nil && g.Pending { - pending = append(pending, n) - } - } - if len(pending) > 0 { - // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on - // a machine since its step was chosen could reach a second machine in it; the step is chosen again. - if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { - return false, err - } else if grew != "" { - for _, n := range pending { - delete(r.Gates, n) - } - opened, waiting, err := openRestStep(ctx, open, p) - if err != nil { - return true, err - } - p.Note = "chosen again: " + grew - if !opened && waiting != "" { - p.Note += "; " + waiting - } - return true, nil - } - var sent, already, waiting []string - for _, node := range pending { - out, err := sendKept(ctx, open, p, node, r.Gates[node], now) - if err != nil { - return true, err - } - switch out { - case keptSent: - sent = append(sent, node) - case keptAlready: - already = append(already, node) - case keptNothing: - delete(r.Gates, node) - r.Done = append(r.Done, node) - case keptWaiting: - waiting = append(waiting, node) - } - } - var said []string - if len(sent) > 0 { - said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there") - } - if len(already) > 0 { - said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send") - } - if len(waiting) > 0 { - said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries") - } - if len(said) > 0 { - p.Note = strings.Join(said, "; ") - } - return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil - } - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return false, err - } - byNode := map[string]inventory.Reported{} - for _, rep := range reports { - byNode[rep.Node] = rep - } - changed, noteWas := false, p.Note - var notes, judging []string - batched, back := batchingRollbacks(ctx) - for _, node := range r.Order { - g := r.Gates[node] - if g == nil || g.Verdict != "" || g.Silent { - continue - } - // What another machine's failure already put back is not this gate's to judge (ADR 0353 §3). - if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) { - decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now) - r.Failed = append(r.Failed, node) - changed = true - continue - } - // **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged - // until it reports, and past ApplySilentAfter it is listed and left. - outcome, reported := byNode[node].Outcome, reportedOnSend(byNode[node], node, g) - if sent, kept := g.Sent[node]; kept && !gateFirstMachine { - // A module handed to the provisioner is applied only once its client said so too (ADR 0362, issue 840). - carried := make([]string, 0, len(g.Carried)) - for _, c := range g.Carried { - carried = append(carried, c.Module) - } - outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx)) - } - if !reported { - if now.Sub(*g.Since) > applySilentAfter { - g.Silent = true - r.Silent = append(r.Silent, node) - changed = true - notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node, - applySilentAfter)) - continue - } - judging = append(judging, node+": not yet reported on its send") - continue - } - if !gateFirstMachine { - // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and - // fails the walk, with nothing put back — apply-failing says it. - switch outcome { - case inventory.OutcomeApplied: - decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) - passCarried(ctx, open, p, g, "") - r.Done = append(r.Done, node) - changed = true - case inventory.OutcomeFailed, inventory.OutcomeRefused: - decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, outcome), now) - r.Failed = append(r.Failed, node) - changed = true - notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, outcome)) - default: - if strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed) { - decide(g, inventory.GateFailed, fmt.Sprintf("%s: %s", node, outcome), now) - r.Failed = append(r.Failed, node) - changed = true - notes = append(notes, fmt.Sprintf("%s: %s: nothing put back, apply-failing says it", node, outcome)) - continue - } - judging = append(judging, node+": not yet reported on its send") - } - continue - } - verdict, err := judgeMoves(ctx, open, g, nil, now) - if err != nil { - return changed, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, g, "", nil) { - changed = true - notes = append(notes, p.Note) - } - judging = append(judging, node+": "+gateLine(g)) - case inventory.GatePassed: - passCarried(ctx, open, p, g, "") - r.Done = append(r.Done, node) - changed = true - case inventory.GateSuperseded: - // Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on - // it; what still waits is released again. - r.Done = append(r.Done, node) - changed = true - notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why)) - default: - // **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other - // machine of the step goes on to its own verdict. - stateWas := p.State - p.Note = "" - failCarried(batched, open, p, g, "") - p.State = stateWas - r.Failed = append(r.Failed, node) - changed = true - notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note)) - returnedElsewhere(ctx, open, r, node, g) - } - } - p.Note = "" - sendRollbacks(ctx, open, p, back) - if p.Note != "" { - notes = append(notes, strings.TrimPrefix(p.Note, "; ")) - } - if len(judging) > 0 { - note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ") - p.Note = note - return changed || noteWas != note, nil - } - // Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3). - if len(r.Failed) == 0 { - opened, waiting, err := openRestStep(ctx, open, p) - if err != nil { - return changed, err - } - if opened { - return true, nil - } - if waiting != "" { - note := strings.Join(append(notes, waiting), "; ") - p.Note = note - return changed || noteWas != note, nil - } - } - p.Tier = len(p.Tiers) - end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) - if len(r.Silent) > 0 { - end += "; silent on their send, left: " + strings.Join(r.Silent, ", ") - } - if len(r.Skipped) > 0 { - end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - p.State = inventory.PlanDone - if len(r.Failed) > 0 { - p.State = inventory.PlanFailed - end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end - } else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) { - // **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it. - p.State = inventory.PlanFailed - end = "no machine of the rest reported on its send; " + end - } - p.Note = strings.Join(append(notes, end), "; ") - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; -// empty when none would. -func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { - if !gateFirstMachine { - return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return "", err - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - for _, node := range pending { - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - continue // sendKept says it waits - } - if err != nil { - return "", err - } - kept := r.Gates[node].Carried - for _, mv := range moves { - // The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9). - if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { - return c.Module == mv.Module && sameCommit(c.To, mv.To) - }) { - continue - } - return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil - } - } - return "", nil -} - -// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the -// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. -func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { - if r.At == nil { - return false - } - if s, kept := g.Sent[node]; kept { - return s.ReportsOn(r) - } - return r.Current -} - -// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's -// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3). -func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) { - for _, c := range g.Carried { - if c.Build == "" { - continue - } - if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked { - continue - } - for node, other := range r.Gates { - if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) { - continue - } - for _, oc := range other.Carried { - if oc.Module == c.Module && sameCommit(oc.To, c.To) { - other.Returned = append(other.Returned, c.Module) - } - } - } - } -} - -// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2): -// every machine left that is heard from and has something waiting, except that a build no gate has passed goes -// to one machine of the step only. Answers whether a step was kept, or why the walk waits. -func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) { - r := p.Release - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, "", err - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return false, "", err - } - waiting := map[string][]inventory.CarriedMove{} - var nodes, walked []string - for _, node := range r.Order[min(r.Next, len(r.Order)):] { - if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) || - slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) { - continue - } - if !heard[node] { - r.Skipped = append(r.Skipped, node) - continue - } - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - walked = append(walked, fmt.Sprintf("%s (%v)", node, err)) - continue - } - if err != nil { - return false, "", fmt.Errorf("reading what waits on %s: %w", node, err) - } - if len(moves) == 0 { - r.Done = append(r.Done, node) - continue - } - waiting[node] = moves - nodes = append(nodes, node) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return false, "", err - } - // **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first. - step := nodes - if gateFirstMachine { - step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) - } - if len(step) == 0 { - if len(walked) > 0 { - return false, "waiting before " + strings.Join(walked, "; "), nil - } - return false, "", nil - } - if r.Gates == nil { - r.Gates = map[string]*inventory.PlanGate{} - } - for _, node := range step { - r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true} - } - p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", ")) - if left := len(nodes) - len(step); left > 0 { - p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left) - } - return true, "", nil -} - -// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the -// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. -func oneAtATime(module string, m catalogue.Manifest) bool { - if module == hostModule { - return true - } - return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) -} - -// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless -// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's -// first machine. Pure. -// -// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a -// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus -// shows only as its silence and no rollback reaches it. -func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, - alone func(module string) bool) []string { - first := map[string]string{} - var out []string - for _, n := range nodes { - var unpassed []string - for _, mv := range waiting[n] { - if !passed(mv.Module, mv.To) || alone(mv.Module) { - unpassed = append(unpassed, mv.Module+"@"+mv.To) - } - } - if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) { - continue - } - for _, k := range unpassed { - first[k] = n - } - out = append(out, n) - } - return out -} - -// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 -// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. -func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) { - recent, err := inv.RecentPlans(ctx, 20) - if err != nil { - return nil, err - } - for _, p := range recent { - if p.Release == nil || p.Release.Settings != nil || p.Open() { - continue - } - if len(p.Release.Silent) == 0 { - return nil, nil - } - machines := append([]string(nil), p.Release.Silent...) - sort.Strings(machines) - o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent", - Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines, - Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+ - "left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)} - w := releaseSilentWords(machines) - o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs - return []conditions.Observation{o}, nil - } - return nil, nil -} - -// backlogObservation is what the gate's probe says of a release held after a failure. -func backlogObservation() []conditions.Observation { - if backlogNow.held == "" || len(backlogNow.waiting) == 0 { - return nil - } - n := 0 - var machines []string - seen := map[string]bool{} - var modules []string - for node, moves := range backlogNow.waiting { - n += len(moves) - machines = append(machines, node) - for _, mv := range moves { - if !seen[mv.Module] { - seen[mv.Module] = true - modules = append(modules, mv.Module) - } - } - } - sort.Strings(machines) - sort.Strings(modules) - o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held", - Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines, - Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+ - "them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)} - w := releaseHeldWords(modules, machines) - o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs - // Asked of the operator, who releases them with an Approve (novox/hq issue 1732): the controller's own - // `upgrade release-backlog`, which opens the release plan that walks them through the machines, each judged. - o.Actions = []conditions.Action{{Label: "Release", Verb: "mesh-controller.upgrade", Level: conditions.LevelApprove, - Arguments: map[string]string{"release-backlog": "true", "why": "", "cause": conditions.CauseOperatorAnswer}}} - return []conditions.Observation{o} -} - -// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`. -func backlogCommand(ctx context.Context, sub string, args []string) error { - set := flag.NewFlagSet("upgrade "+sub, flag.ContinueOnError) - why := addHandActFlags(set) - if rest, err := parseAround(set, args); err != nil { - return err - } else if len(rest) > 0 { - return errors.New("upgrade backlog | upgrade release-backlog --why ") - } - if sub == "release-backlog" { - if err := why.require("upgrade release-backlog"); err != nil { - return err - } - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - if sub == "release-backlog" { - // Said on the bus (novox/hq ADR 0308 decision 1): the release plan starts walking here, and its start is - // said by the verb that opens it. - var p *inventory.Plan - if err := sayingOnTheBus(ctx, open.inventory, func() error { - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - return err - } - defer release() - why.record(ctx, "upgrade release-backlog", nil) - p, err = releaseBacklog(ctx, open, link.Caller()) - return err - }); err != nil { - return err - } - if p == nil { - fmt.Println("nothing waits for a gate on any machine heard from: nothing to release") - return nil - } - fmt.Printf("%s releases it; `plans %s` says where it is\n", p.ID, p.ID) - return nil - } - waiting, err := waitingMoves(ctx, open, false) - if err != nil { - return err - } - if len(waiting) == 0 { - fmt.Println("no build waits for a gate on any machine") - return nil - } - nodes := make([]string, 0, len(waiting)) - for n := range waiting { - nodes = append(nodes, n) - } - sort.Strings(nodes) - for _, n := range nodes { - fmt.Printf("%s: %d build(s) wait for a gate\n", n, len(waiting[n])) - for _, mv := range waiting[n] { - fmt.Printf(" %-28s %s → %s\n", mv.Module, short(mv.From), short(mv.To)) - } - } - return nil -} - -// sayRecreations says, on each move a send carries, what it does to the module's containers (novox/hq -// ADR 0245): the build the machine ran against the one it is sent, container by container. Left unsaid -// for a move whose earlier build is not in the records. -func sayRecreations(ctx context.Context, open *stores, moves []inventory.CarriedMove) { - if len(moves) == 0 { - return - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return - } - for i, mv := range moves { - to, held := shelf[mv.Module] - if !held || mv.From == "" { - continue - } - from, found, err := open.inventory.ManifestAt(ctx, mv.Module, mv.From) - if err != nil || !found { - continue - } - moves[i].Recreates = catalogue.Recreates(from, to).Say(mv.Module) - } -} - -// sayWhatAPushRecreates says, per machine a push is about to send, every module the send moves and what -// the move recreates (novox/hq ADR 0245) — the same words a gated send keeps on its plan. A person's push -// carries every module's new build, whatever its policy and whether or not a gate saw it, so it is the -// send that most needs to say so: on 2026-10-07 a `push` of one machine moved mail to a new build and -// recreated one of its containers with a new image, and the answer said nothing about it. A machine whose -// last send's builds are not known is not said, and neither is a failure to read: the push goes on. -func sayWhatAPushRecreates(ctx context.Context, open *stores, sending []readyNode, w io.Writer) { - if len(sending) == 0 { - return - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return - } - for _, r := range sending { - sent, known, err := open.inventory.SentBuilds(ctx, r.node) - if err != nil || !known { - continue - } - moves := pushMoves(f, r.node, r.declared.Builds, sent) - if len(moves) == 0 { - continue - } - sayRecreations(ctx, open, moves) - fmt.Fprintf(w, "%s moves %d module(s):\n", r.node, len(moves)) - for _, mv := range moves { - said := mv.Recreates - if said == "" { - said = "recreates none of its containers, or what it ran is not in the records" - } - fmt.Fprintf(w, " %-28s %s → %s: %s\n", mv.Module, short(mv.From), short(mv.To), said) - } - } -} - -// pushMoves is what a send carrying these builds moves on a machine last sent `sent`: every module it ran -// before at a build not identical to the one it is now sent. A module new to the machine is not a move. -func pushMoves(f moveFacts, node string, carries, sent map[string]string) []inventory.CarriedMove { - var out []inventory.CarriedMove - for m, to := range carries { - was, ran := sent[m] - if !ran || was == "" || to == "" || f.identical(m, was, to) { - continue - } - out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: to}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - -// recreationsSaid is every recreation a send's moves say, joined: what a plan's note carries. -func recreationsSaid(moves []inventory.CarriedMove) string { - var said []string - for _, mv := range moves { - if mv.Recreates != "" { - said = append(said, fmt.Sprintf("on %s %s", mv.Node, mv.Recreates)) - } - } - return strings.Join(said, "; ") -} - -// The grants step (novox/hq issue 490). -// -// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the -// declaration of the machine holding the bus. A send to any other machine that changed it put that machine -// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On -// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it -// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name -// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The -// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there -// unjudged. -// -// So before a gated send, when the user list composed now is not the one the machine holding the bus was -// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new -// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said -// with why and passed over: the send goes on as it did before, and its gate says what it finds. - -// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store. -var brokerBehindOf = brokerBehind - -// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the -// list changed; answers what it did, or nil when there was nothing to send. -func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep { - holder, behind, err := brokerBehindOf(ctx, open, []string{node}) - if err != nil { - fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err) - return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()} - } - if holder == "" || holder == node || !behind { - // No bus with a user list, the gate's own machine holds it (its send carries the list first), or the - // list is the one already sent. - return nil - } - if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil { - fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+ - "it — the bus may refuse what the send newly grants: %v\n", holder, node, err) - return &inventory.GrantsStep{Node: holder, Failed: err.Error()} - } - now := time.Now().UTC() - fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n", - holder, node) - return &inventory.GrantsStep{Node: holder, At: &now} -} - -// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list. -var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent") - -// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding -// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else. -// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused -// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment -// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of -// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary -// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that -// was never sent is a new assignment, refused whatever its resources. -func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) { - inv := open.inventory - sent, known, err := inv.SentBuilds(ctx, holder) - if err != nil { - return "", err - } - if !known { - return fmt.Sprintf("what %s was last sent is not known", holder), nil - } - var other []string - listID := "" - for _, m := range plan.Modules { - // A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent" - // only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment. - if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" { - other = append(other, m.Module+" newly assigned") - } - if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) { - listID = m.Module + "." + catalogue.BusUsersID() - } - } - var facts *moveFacts - for m, was := range sent { - now, carried := declared.Builds[m] - if !carried || sameCommit(now, was) { - continue - } - if facts == nil { - f, err := readMoveFacts(ctx, inv) - if err != nil { - return "", err - } - facts = &f - } - if !facts.identical(m, was, now) { - other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now))) - } - } - for _, f := range declared.foreseen { - other = append(other, f+" would be minted") - } - body, err := declared.Body() - if err != nil { - return "", err - } - after, err := summarize(body) - if err != nil { - return "", err - } - prev, err := inv.SentSummary(ctx, holder) - if err != nil { - return "", err - } - if len(prev) == 0 { - return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil - } - var before []sentResource - if err := json.Unmarshal(prev, &before); err != nil { - return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err) - } - other = append(other, otherThanTheList(diffSent(before, after), listID)...) - sort.Strings(other) - if len(other) == 0 { - return "", nil - } - return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil -} - -// otherThanTheList is what a diff against the last send changes besides the user list's content: every -// resource added, removed or changed, but the list's own resource when its content is all that changed — -// its path, mode and every other field must be as last sent (novox/hq issue 490). -func otherThanTheList(d sentDiff, listID string) []string { - var other []string - for _, id := range d.Added { - other = append(other, "+"+id) - } - for _, id := range d.Removed { - other = append(other, "-"+id) - } - for _, c := range d.Changed { - if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" { - continue - } - other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", "))) - } - return other -} - -// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged. -func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) { - grants := grantsStep(ctx, open, node) - sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node}) - return sent, grants, err -} - -// grantsSaid is a grants step as `plans` and a walk's note say it. -func grantsSaid(g *inventory.GrantsStep) string { - switch { - case g == nil: - return "" - case g.Failed != "": - return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed) - default: - return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node) - } -} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go deleted file mode 100644 index 9ed917a6..00000000 --- a/cmd/mesh-controller/release_plan.go +++ /dev/null @@ -1,2474 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "slices" - "sort" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A merge produces a tiered plan the mesh keeps (novox/hq ADR 0162). -// -// The handler that hears the merge computes the plan from the catalogue's one dependency relation, -// writes it to the store, asks the first tier and returns — the receive loop is never held by a -// build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can -// move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report); -// a controller replaced mid-plan finds the plan where it left it and looks first. - -// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since -// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on. -var gateFirstMachine = false - -// planWaitBound is how long a plan may wait on one thing before `status` names it red. -const planWaitBound = 30 * time.Minute - -// tiersOf sorts a set of modules into tiers along the ordering edges among them: tier 0 depends -// on nothing else in the set, tier 1 only on tier 0, and so on. An edge to a module outside the set says -// nothing about the order inside it. A cycle — which the catalogue should never produce — puts -// what remains in one last tier rather than losing it, and is said by the caller. -func tiersOf(set []string, edges []inventory.Edge) [][]string { - in := map[string]bool{} - for _, m := range set { - in[m] = true - } - deps := map[string]map[string]bool{} - for _, m := range set { - deps[m] = map[string]bool{} - } - for _, e := range edges { - // A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's - // build needs nothing of A's first. Stands-on and declared order after the base is built, and the - // build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206). - if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages { - continue - } - // **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it. - if e.Kind == inventory.EdgeBuiltBy { - continue - } - deps[e.From][e.To] = true - } - placed := map[string]bool{} - var tiers [][]string - for len(placed) < len(set) { - var tier []string - for _, m := range set { - if placed[m] { - continue - } - free := true - for d := range deps[m] { - if !placed[d] { - free = false - break - } - } - if free { - tier = append(tier, m) - } - } - if len(tier) == 0 { - // A cycle: everything left, together, and the caller says so. - for _, m := range set { - if !placed[m] { - tier = append(tier, m) - } - } - } - sort.Strings(tier) - for _, m := range tier { - placed[m] = true - } - tiers = append(tiers, tier) - } - return tiers -} - -// reachableFrom is the moved modules plus everything that depends on them, through every layer: -// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine -// is not changed by a new build machine, so a built-by edge orders and gates a plan and never -// widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change. -func reachableFrom(moved []string, edges []inventory.Edge) []string { - in := map[string]bool{} - for _, m := range moved { - in[m] = true - } - for grew := true; grew; { - grew = false - for _, e := range edges { - // Built-by and worker-of order a plan; neither widens it. A new build machine changes - // nothing it builds, and a new controller changes nothing about the holder it orders. A - // packages edge, read from a record made before novox/hq ADR 0267, widens nothing either: - // a shared file moves each module whose build source holds it, directly. - if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages || - e.Kind == edgeChangeOrder { - continue - } - if in[e.To] && !in[e.From] { - in[e.From] = true - grew = true - } - } - } - out := make([]string, 0, len(in)) - for m := range in { - out = append(out, m) - } - sort.Strings(out) - return out -} - -// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages -// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is -// no cycle, and saying one was is a false report in every such plan's log. -func hasCycle(tiers [][]string, edges []inventory.Edge) bool { - if len(tiers) == 0 { - return false - } - last := map[string]bool{} - for _, m := range tiers[len(tiers)-1] { - last[m] = true - } - for _, e := range edges { - if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy { - continue - } - if last[e.From] && last[e.To] { - return true - } - } - return false -} - -// planOfMerge is the plan one merge produces: the moved modules and everything reachable from them, -// tiered, with the merge it answers. -func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inventory.Plan { - p := planOfMoves(moved, edges) - merged, _ := time.Parse(time.RFC3339Nano, m.MergedAt) - p.Repository, p.Branch, p.Commit, p.Merged = m.Owner+"/"+m.Repo, m.Base, m.Commit, merged.UTC() - return p -} - -// planOfMoves is the walk of a set of moved modules (novox/hq ADR 0162, 0276): they and everything reachable -// from them, tiered along the graph, with no merge named yet. -func planOfMoves(moved []string, edges []inventory.Edge) inventory.Plan { - set := reachableFrom(moved, edges) - tiers := tiersOf(set, edges) - modules := map[string]*inventory.PlanModule{} - for _, name := range set { - modules[name] = &inventory.PlanModule{} - } - return inventory.Plan{ - ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()), - Created: time.Now().UTC(), - State: inventory.PlanBuilding, - Tiers: tiers, - Modules: modules, - } -} - -// gates is what the next tier needs running from this one: a module of the tier that a later -// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by -// the machines running it before the next tier is asked. A base an image stands on need only be -// built; a source another module packages need not even be that. -func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - inTier := map[string]bool{} - for _, m := range p.Tiers[p.Tier] { - inTier[m] = true - } - later := map[string]bool{} - for _, tier := range p.Tiers[p.Tier+1:] { - for _, m := range tier { - later[m] = true - } - } - seen := map[string]bool{} - var out []string - // The plan's own order (ADR 0419) is not in the catalogue's graph: it is kept on the plan. - if p.Delivery != nil { - edges = append(append([]inventory.Edge{}, edges...), p.Delivery.Order...) - } - for _, e := range edges { - // A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the - // member before is running on its machines before the member after is asked. - // A built-by edge waits for nothing since novox/hq ADR 0360 §3. - ordered := e.Kind == edgeChangeOrder - if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) { - seen[e.To] = true - out = append(out, e.To) - } - } - sort.Strings(out) - return out -} - -// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine -// running it runs (novox/hq issue 280): nothing is sent for it. -// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader. -const ( - noMovePrefix = "no move" - noMoveWhy = noMovePrefix + ": made from the source every machine running it runs" -) - -// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says, -// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648). -func noMove(s *inventory.PlanModule) bool { - return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix) -} - -// applied says whether every machine running the module has reported since the module was built. -// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones -// after that machine was sent the build (novox/hq issue 256). -func appliedEach(module string, since func(node string) time.Time, running []string, reports []inventory.Reported) (bool, []string) { - at := map[string]*time.Time{} - for _, r := range reports { - at[r.Node] = r.At - } - var waiting []string - for _, n := range running { - if t := at[n]; t == nil || t.Before(since(n)) { - waiting = append(waiting, n) - } - } - return len(waiting) == 0, waiting -} - -// gateNeed is a module the next tier waits for: the machines running it, and from when each must have reported. -type gateNeed struct { - module string - running []string - since func(node string) time.Time -} - -// silentNodes is the machines a walk sent and heard nothing from past ApplySilentAfter, on whichever tier it sent -// them (novox/hq ADR 0360 §7, issue 1160): the one rule both the gates between tiers and the walk's end read, through -// walkReports. A machine that reports on its send is not silent from then on. -func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[string]bool { - _, _, silent := walkReports(p, reports, now, prov) - out := map[string]bool{} - for _, n := range silent { - out[n] = true - } - return out -} - -// withoutSilent is the machines of running that are not silent: a later tier is not sent to a machine the walk heard -// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §7). -func withoutSilent(running []string, silent map[string]bool) []string { - return slices.DeleteFunc(slices.Clone(running), func(n string) bool { return silent[n] }) -} - -// settleTierGate is the wait between tiers on the reports of the machines running what the next tier needs: what is -// still waited for ("module on a, b"), the silent machines left out of the wait, sorted, and whether every machine the -// gate waits on is silent, so that nothing it needs was delivered anywhere (issue 1160). -func settleTierGate(needs []gateNeed, reports []inventory.Reported, silent map[string]bool) (waiting, quiet []string, lost bool) { - all, silentAll := 0, 0 - seenQuiet := map[string]bool{} - seenNode := map[string]bool{} - for _, need := range needs { - for _, n := range need.running { - if !seenNode[n] { - seenNode[n] = true - all++ - if silent[n] { - silentAll++ - } - } - } - _, on := appliedEach(need.module, need.since, need.running, reports) - var still []string - for _, n := range on { - if silent[n] { - if !seenQuiet[n] { - seenQuiet[n] = true - quiet = append(quiet, n) - } - continue - } - still = append(still, n) - } - if len(still) > 0 { - waiting = append(waiting, fmt.Sprintf("%s on %s", need.module, strings.Join(still, ", "))) - } - } - sort.Strings(quiet) - return waiting, quiet, all > 0 && silentAll == all -} - -func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) { - at := map[string]*time.Time{} - for _, r := range reports { - at[r.Node] = r.At - } - var waiting []string - for _, n := range running { - if t := at[n]; t == nil || t.Before(builtAt) { - waiting = append(waiting, n) - } - } - return len(waiting) == 0, waiting -} - -// askTier asks the build machine for the named modules of the plan's tier — all of it, or what of it is still -// unasked (novox/hq issue 754) — and marks each asked. A module -// the catalogue no longer holds ends as deleted (issue 1922); one whose ask could not be made is a failure of the -// plan: a tier half asked is a tier that will never complete. -func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, names []string) error { - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - for _, name := range names { - askModule(ctx, p, name, byName) - } - return nil -} - -// unaskedOf are the modules of a plan's current tier it has still to ask: those it never asked, and one it holds -// built that it never asked for (novox/hq issue 754) — a build taken in by a controller from before a walk took -// only its own asks, somebody else's, which the walk would otherwise send as its own (ADR 0219). A tier partly asked is asked the rest, not -// waited on for ever: a walk that waits for every build of a tier it never asked whole never moves again. -func unaskedOf(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - switch { - case s == nil || s.State == "": - out = append(out, m) - case s.State == "built" && s.Build == "" && s.AskedAt == nil: - out = append(out, m) - } - } - return out -} - -// askedInTier says the plan asked a module of its current tier itself: under an id, or at a time it kept. -func askedInTier(p inventory.Plan) bool { - if p.Tier >= len(p.Tiers) { - return false - } - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && (s.Build != "" || s.AskedAt != nil) { - return true - } - } - return false -} - -// askModule asks the build machine for one module of a plan and marks it asked, with the id it was -// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked. -func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) { - now := time.Now().UTC() - state := p.Modules[name] - if state == nil { - state = &inventory.PlanModule{} - p.Modules[name] = state - } - e, known := byName[name] - if !known { - // **A module the catalogue no longer holds ends as deleted** (issue 1922): the plan's own merge removed it, so - // there is nothing to build or send; it is no failure, and a retry of a plan failed that way resumes. - state.State = planDeleted - state.Why = "no longer in the catalogue" - return - } - // **Every module is taken from the builder's done key, asking no build** (novox/hq ADR 0503 §6, ADR 0501, ADR 0453 §5; - // issue 1851): the look settles it (settleFromKeys). A key not done yet waits, named in the plan, and never fails it. - takeFromKey(p, state, e, now) -} - -// refusalsKeptFor is the refusals kept in this process for these builds (keepRefusalFor), by id. -func refusalsKeptFor(ids []string) map[string]string { - out := map[string]string{} - refusedLeftMu.Lock() - defer refusedLeftMu.Unlock() - for _, id := range ids { - if k, ok := refusedLeft[id]; ok { - out[id] = k.made - } - } - return out -} - -// refusedLeft is why each outcome left to the records was refused, by the build's id: kept in the process beside the -// store's record (RecordRefused), so a store that did not take it still fails the plan while this controller serves. -var ( - refusedLeftMu sync.Mutex - refusedLeft = map[string]madeKept{} -) - -// keepRefusalFor keeps why an outcome left to the records was refused, forgetting refusals older than madeLeftFor. -func keepRefusalFor(id, why string, now time.Time) { - if id == "" || why == "" { - return - } - refusedLeftMu.Lock() - defer refusedLeftMu.Unlock() - for k, v := range refusedLeft { - if now.Sub(v.at) > madeLeftFor { - delete(refusedLeft, k) - } - } - refusedLeft[id] = madeKept{made: why, at: now} -} - -// madeLeft is what each build left to the records did, by the build's id, until a settle takes it: kept in the -// process, so a controller replaced before the settle loses only these words, never the outcome. -var ( - madeLeftMu sync.Mutex - madeLeft = map[string]madeKept{} -) - -type madeKept struct { - made string - at time.Time -} - -// madeLeftFor is how long words nobody settled are kept: a plan that ended or moved on never asks for them. -const madeLeftFor = time.Hour - -// takeMadeOf is what the build of this id did, if its outcome was left to the records with words; taken once. -func takeMadeOf(id string) string { - madeLeftMu.Lock() - defer madeLeftMu.Unlock() - k, ok := madeLeft[id] - if !ok { - return "" - } - delete(madeLeft, id) - return k.made -} - -// startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past -// its wait, not the backlog's — with why, or empty: a walk waits only for a started walk it shares something with -// (novox/hq ADR 0276 decision 4, ADR 0407 decision 4, issue 973). -func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (behind, why string, err error) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return "", "", err - } - started := startedWalks(plans, p.ID, p.Created) - if len(started) == 0 { - return "", "", nil - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return "", "", err - } - onPath, err := ownPathNow(ctx, inv) - if err != nil { - return "", "", err - } - open, err := openWalksOf(ctx, inv, started, onPath) - if err != nil { - return "", "", err - } - mine, err := openWalkOf(ctx, inv, p, onPath) - if err != nil { - return "", "", err - } - behind, why = relatedOpen(mine.fp, mine.own, open, edges) - return behind, why, nil -} - -// refusalNotePrefix and refusalNoteSuffix frame the note of a step the walk was refused: "tier N: — tried -// again". Only a note so framed is a hold (novox/hq issue 920); a tier's summary left in the note is not. -const ( - refusalNotePrefix = "tier " - refusalNoteSuffix = " — tried again" -) - -// refusalNote is the note of a plan whose step at tier was refused with err. -func refusalNote(tier int, err error) string { - return refusalNotePrefix + fmt.Sprint(tier) + ": " + err.Error() + refusalNoteSuffix -} - -// refused says a plan's note is a refusal the walk wrote. -func refused(p inventory.Plan) bool { - return strings.HasPrefix(p.Note, refusalNotePrefix) && strings.HasSuffix(p.Note, refusalNoteSuffix) -} - -// deferredNote begins the note of a walk deferred behind another. -const deferredNote = "let go; starts once " - -// deferred says a walk has its word and has not started: let go while another walk was started, it waits for -// that one to end (startedBeside), and its note says so. Read as a wait, not as a tier running late: `plans` -// and `status` say its note, and S3 leaves it out. A let-go walk whose first ask failed carries that error as -// its note instead, and is watched as before. -func deferred(p inventory.Plan) bool { - return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Delivery != nil && - p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote) -} - -// currentTierBuilt says every module of a plan's current tier is built (or deleted), so no build is what it waits on. -func currentTierBuilt(p inventory.Plan) bool { - if p.Tier < 0 || p.Tier >= len(p.Tiers) || len(p.Tiers[p.Tier]) == 0 { - return false - } - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - if s == nil || (s.State != "built" && s.State != planDeleted) { - return false - } - } - return true -} - -// askedAny says a plan asked any module. -func askedAny(p inventory.Plan) bool { - for _, s := range p.Modules { - if s != nil && s.State != "" { - return true - } - } - return false -} - -// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold -// on the plans so it never lengthens it: measured, never acted on, and an error only said. -func keepWalkPhases(ctx context.Context, open *stores) { - if err := recordWalkPhases(ctx, open.inventory, time.Now()); err != nil { - fmt.Printf("plans: the phases of the walks ended lately could not be kept: %v\n", err) - } -} - -// advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules -// are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says -// whether a step failed, which the look at the plans (lookAtPlans) retries. -// -// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the -// old and the new while a machine hands its controller over — would each ask a tier the other had just asked. -func advanceHeld(ctx context.Context, open *stores) (failed bool) { - inv := open.inventory - // A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407). - defer func() { - if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil { - fmt.Printf("batches: %v\n", err) - } - }() - plans, err := inv.OpenPlans(ctx) - if err != nil { - fmt.Printf("plans: cannot read them: %v\n", err) - return true - } - if len(plans) == 0 { - return false - } - edges, err := inv.Dependencies(ctx) - if err != nil { - fmt.Printf("plans: cannot read the dependencies: %v\n", err) - return true - } - rollsOut := func(module string) bool { - u, err := inv.UpgradeOf(ctx, module) - return err == nil && u.RollOut - } - for i := range plans { - p := &plans[i] - for p.Open() { - // **Saved only when the step changed it** (novox/hq issue 296). Every step was saved, and - // the steps run on every tick and after every build outcome on the mesh: a plan whose tier - // stood still for half an hour was written every few seconds, its revision moved, `plan-moved` - // said on the bus, and its last save read as when it began building. - before := planSnapshot(*p) - sends := &sendsMade{} - moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut) - if err != nil { - failed = true - // Kept in the plan, so `plans` says why it has not moved rather than the log alone; - // the state is left as it was and the step is tried again by the next look. Said and - // kept when it is new: the same refusal on every tick is one fact, not one per tick. - p.Note = refusalNote(p.Tier, err) - // A refusal for the bus's planned step is a person's to end: S17 says it from its first moment. - if refusedForTheBus(err) { - busRefusedFirst.mark(p.ID, time.Now()) - } - if planSnapshot(*p) != before { - fmt.Printf("%s: %v\n", p.ID, err) - if err := inv.SavePlan(ctx, p); err != nil { - fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) - } - } - break - } - if p.State == inventory.PlanFailed { - sayUnsent(p, rollsOut) - } - // A module the bus did not answer an ask of is asked again by a look, and no event may bring one (a tier all - // refused has no build in flight): the look fails, so the looker retries it from five seconds (issue 1334). - if busWaiting(*p) { - failed = true - } - if planSnapshot(*p) != before { - if err := inv.SavePlan(ctx, p); err != nil { - fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) - failed = true - break - } - // Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent. - saySends(ctx, *p, sends) - } - if !moved { - break - } - } - } - return failed -} - -// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store -// stamps itself (when it was saved, its revision and epoch, when its tier was entered) is left out. -func planSnapshot(p inventory.Plan) string { - p.Updated, p.Revision, p.Epoch, p.TierEntered = time.Time{}, 0, 0, time.Time{} - b, err := json.Marshal(p) - if err != nil { - // Unreadable is never the same: the plan is saved, as every step was before. - return fmt.Sprintf("unmarshallable %p %v", &p, err) - } - return string(b) -} - -// advanceOnce takes one step of one plan and says whether anything changed. -func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, - edges []inventory.Edge, rollsOut func(string) bool) (bool, error) { - inv := open.inventory - if p.Release != nil { - return advanceRelease(ctx, open, p) - } - // **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239): nothing of it is - // registered before its turn, so no other send carries it to a machine. - if p.Waiting() { - note := waitingNote(*p) - changed := p.Note != note - p.Note = note - return changed, nil - } - if p.Tier >= len(p.Tiers) { - p.State = inventory.PlanDone - fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers)) - return true, nil - } - tier := p.Tiers[p.Tier] - // Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754). - if unasked := unaskedOf(*p); len(unasked) > 0 { - // **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): a walk - // about to ask its first tier while a started walk it is related to is open waits for that one to end, - // let go or not, and says so and why. A started walk it shares nothing with does not hold it. - if p.Tier == 0 && !askedInTier(*p) { - if behind, why, err := startedBeside(ctx, inv, *p); err != nil { - return false, err - } else if behind != "" { - note := fmt.Sprintf("%s%s ended — %s", deferredNote, behind, why) - changed := p.Note != note - p.Note = note - return changed, nil - } - } - // **A backup before the send** (novox/hq ADR 0404, issue 1065): a walk that starts asks the backup holders of - // the machines running a moved module that keeps data, and asks its first tier only when every one is done. - if p.Tier == 0 && !askedInTier(*p) { - if proceed, err := backupsFirst(ctx, inv, p); err != nil { - return false, err - } else if !proceed { - return false, nil - } - } - if len(unasked) < len(tier) { - fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", ")) - } - note := p.Note - if err := askTier(ctx, inv, p, unasked); err != nil { - return false, err - } - // Nothing asked and nothing new to say (the bus still silent, issue 1334): not saved again at every look. - return len(unaskedOf(*p)) < len(unasked) || p.Note != note || p.State != inventory.PlanBuilding, nil - } - // **Asked: settle from the build records first** (novox/hq 04-ISSUES/214). An outcome is taken - // in by whichever controller hears it, and a merge to the controller's own repository replaces - // the controller in its first tier: the build that produced the new one is recorded, and the - // plan never hears it. The record is the fact; a build recorded after the ask is that tier's - // outcome, whoever was listening. - // **Taken from the builder's keys first** (ADR 0503 §6, issue 1851): every module of the tier but the builder's own. - if keyed(p, tier) { - keys, closeKeys, err := openPlanKeys() - if err != nil { - return false, err - } - was := p.State - changed, err := settleFromKeys(ctx, open, keys, p, tier) - closeKeys() - if err != nil { - return false, err - } - if p.State == inventory.PlanFailed && was != inventory.PlanFailed { - sayUnsent(p, rollsOut) - } - if changed { - return true, nil - } - } - recorded, byID, err := recordsOfAsked(ctx, inv, p, tier) - if err != nil { - return false, err - } - before, was := statesOf(p), p.State - if settleFromRecords(p, tier, recorded, byID) { - // As taking the outcome directly does (planBuiltSaying): a module deleted at its source is forgotten, and a - // failed plan says what it built and never sent (issue 1708). - deleted, failed := settledEffects(before, p, was) - for _, m := range deleted { - forgetDeleted(ctx, inv, m, p) - } - if failed { - sayUnsent(p, rollsOut) - } - return true, nil - } - // Asked: wait for every build. - var latest time.Time - for _, m := range tier { - s := p.Modules[m] - if s == nil || (s.State != "built" && s.State != planDeleted) { - return false, nil - } - if s.BuiltAt != nil && s.BuiltAt.After(latest) { - latest = *s.BuiltAt - } - } - // Built: send every module of the tier whose policy rolls out, once, to the machines running - // it — whether or not its source commit moved. A dependent rebuilt because its base moved, or - // a module that packages another repository's source, keeps its commit; the catalogue announces - // no move for it and its machines would keep the old image until somebody pushed (novox/hq - // issue 189). A module whose policy records is built and left, as its policy says. - // - // **One machine first, unless the module's policy says together** (novox/hq issue 249, ADR - // 0218). The plan sent every machine running the module at once, and the operator's policy — - // one at a time, stopping at the first that fails, which an announced upgrade honours — was not - // read here at all: a module whose new declarations broke it broke everywhere in the same - // minute. Now the first machine is sent, the plan records it and waits for that machine's report - // after the send to say it applied what it was sent; only then are the rest sent. A first machine - // that fails or refuses stops the module's rollout and the plan with it, the rest untouched. - // - // **One send per machine for the whole tier** (novox/hq issue 281). A send carries the machine's - // whole declaration (ADR 0221), and the plan sent the first machine once per module: on 2026-10-06 a - // tier of seventy modules sent one laptop a new declaration every few seconds, the node-engine set - // aside one for the next, the node tools never settled, and the gate blamed a module for the machine - // being behind. Now every module of the tier whose first machine is the same is sent there in one - // send, judged there by one gate — kept on the first of them, the others gated by it — and the rest - // of the tier's machines are sent once, together, when the gates have passed. - // **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §7, issue 1160): the - // walk's end names it, and the gates between tiers and the sends of the tiers after do not wait for it or go to it. - allReports, err := inv.LastReports(ctx) - if err != nil { - return false, err - } - silent := silentNodes(*p, allReports, time.Now().UTC(), provLookup(ctx)) - // **A machine away when a tier is sent is not sent to and not waited for** (ADR 0435 §1, issue 1161): read once, - // for a tier that has something left to send. - var away map[string]bool - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.SentAt == nil && s.FirstAt == nil && s.State != planDeleted { - away = judgedAway(ctx, open, time.Now().UTC()) - break - } - } - var pending []string - firstOn := map[string][]string{} - runningOf := map[string][]string{} - var rest []string - restTo := map[string][]string{} - together := map[string]bool{} - for _, m := range tier { - state := p.Modules[m] - if state == nil || state.SentAt != nil || state.State == planDeleted { - continue - } - // **A recorded module goes where its move recreates nothing and moves no data** (novox/hq ADR 0377): judged - // per machine against the build it was last sent, said per machine, and sent only there. - var goes map[string]bool - if !rollsOut(m) { - out, err := judgeRecordedInWalk(ctx, inv, p, m, state) - if err != nil { - return false, err - } - if len(out) == 0 { - continue - } - goes = map[string]bool{} - for _, n := range out { - goes[n] = true - } - } - running, err := inv.Running(ctx, m) - if err != nil { - return false, err - } - if goes != nil { - running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] }) - } - if state.FirstAt == nil { - running = withoutSilent(running, silent) - passed := withoutSilent(running, away) - if len(passed) < len(running) { - for _, n := range running { - if away[n] && !slices.Contains(p.Away, n) { - fmt.Printf("%s: %s is away; %s is not sent to it and not waited for\n", p.ID, n, m) - } - } - rememberAway(p, running, away) - running = passed - } - } - runningOf[m] = running - // **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source - // of the build every machine running it was sent, it is registered with that build's artifacts — - // nothing to send, and nothing for a gate to judge. - if state.FirstAt == nil { - if same, err := unchangedOnEvery(ctx, inv, m, state.Build, running); err != nil { - return false, err - } else if same { - now := time.Now().UTC() - state.SentAt = &now - state.Why = noMoveWhy - fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m) - continue - } - } - policy, err := inv.UpgradeOf(ctx, m) - if err != nil { - return false, err - } - // **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine - // running it in one step, with no first machine and no gate, whatever module it is. A module a controller - // before ADR 0360 already sent to its first machine finishes under its gate. - if !gateFirstMachine && state.FirstAt == nil { - policy.Together = true - } - var reports []inventory.Reported - if !policy.Together { - // Read for the choice of the first machine as well as for its report. - if reports, err = inv.LastReports(ctx); err != nil { - return false, err - } - } - now := time.Now().UTC() - // **Sent with others, judged with them** (issue 281): the gate of the send that carried it is its - // verdict on its first machine. A failure there stopped the plan already. - if state.GatedBy != "" { - lead := p.Modules[state.GatedBy] - if lead == nil || lead.Gate == nil || lead.Gate.Verdict == "" { - pending = append(pending, fmt.Sprintf("%s judged with %s on %s", m, state.GatedBy, strings.Join(state.First, ", "))) - continue - } - if lead.Gate.Verdict != inventory.GatePassed { - continue - } - passedWith(m, state, state.GatedBy, lead.Gate) - } - // Read after its pass is taken over from the send that carried it, so a passed gate is never judged - // again from the first machine's later reports (novox/hq issue 335). - step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound) - switch { - case step.failed != "": - // The first machine refused or failed what it was sent, or never said: the gate failed, and - // what it carried is put back there (novox/hq ADR 0236); the rest are left as they were. - failFirstSend(ctx, open, p, m, state, firstRunning(state.First, running), step.failed, step.rest) - return true, nil - case step.waiting != "": - at := "" - if state.FirstAt != nil { - at = state.FirstAt.Local().Format("15:04") - } - pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, at)) - continue - } - // **The gate** (novox/hq ADR 0236, to-be 45 §8): the first machine reported the build applied; - // it is judged by its health before anything else is sent — the rest, or, where it is the only - // machine, the plan's next step. - if state.GatedBy == "" && !policy.Together && state.FirstAt != nil && len(state.First) > 0 { - verdict, err := judgeGate(ctx, open, p, m, state, running, now) - if err != nil { - return false, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, state.Gate, m, state) { - return true, nil - } - pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m, - strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate))) - continue - case inventory.GateFailed: - failFirstSend(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why, step.rest) - return true, nil - case inventory.GateSuperseded: - // Another send moved this module on its first machine (novox/hq issue 352): the build is not - // judged, not marked, not put back; the plan ends here, said, and a newer plan carries on. - state.Why = "superseded: " + state.Gate.Why - p.State = inventory.PlanSuperseded - p.Note = fmt.Sprintf("%s's judging on %s was superseded: %s", m, strings.Join(state.Gate.Machines, ", "), - state.Gate.Why) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - case inventory.GatePassed: - if !state.Gate.Kept { - gatePassed(ctx, open, p, m, state) - state.Gate.Kept = true - } - } - } - switch { - case len(step.send) == 0: - // No machine runs it: nothing to send, and nothing to wait for. - state.SentAt = &now - continue - case step.first: - firstOn[step.send[0]] = append(firstOn[step.send[0]], m) - continue - } - rest = append(rest, m) - restTo[m] = step.send - together[m] = policy.Together - } - // The first machines: one send each for everything of the tier that goes there first, one machine - // per step — the plan is kept between them, so the next machine's send sees what this one walks. - if len(firstOn) > 0 { - machines := make([]string, 0, len(firstOn)) - for n := range firstOn { - machines = append(machines, n) - } - sort.Strings(machines) - for _, node := range machines { - err := firstSend(ctx, open, p, node, firstOn[node], runningOf) - if errors.Is(err, errWalkedElsewhere) { - // A build this machine would carry is being judged elsewhere: it goes once that gate passed. - pending = append(pending, fmt.Sprintf("%s for %s: %v", node, strings.Join(firstOn[node], ", "), err)) - continue - } - if err != nil { - // Not marked sent, so the next step tries again (issue 249): a grant that could not be - // issued is a send that did not happen. - return false, fmt.Errorf("sending %s to %s first after tier %d: %w", strings.Join(firstOn[node], ", "), - node, p.Tier, err) - } - return true, nil - } - } - // The rest, after their gates passed, and what rolls out together: every machine once. - if len(rest) > 0 { - var machines []string - scope := sendScope{modules: map[string]bool{}} - for _, m := range rest { - for _, n := range restTo[m] { - if !slices.Contains(machines, n) { - machines = append(machines, n) - } - } - if together[m] { - // What its policy rolls out together may move wherever it goes; anything else no gate has - // seen still stops the send. - scope.modules[m] = true - } - } - sort.Strings(machines) - was := map[string]map[string]chain.Ref{} - for _, n := range machines { - was[n] = ranBefore(ctx, inv, n) - } - sent, err := sendRollout(withScope(sendingAs(ctx, p), scope), open, machines) - if err != nil { - return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "), - strings.Join(machines, ", "), p.Tier, err) - } - now := time.Now().UTC() - // What each machine of the rest was sent, kept for when it reports it applied (novox/hq ADR 0282 decision - // 6): measured, never acted on. - sentWhat := sentNow(ctx, open.inventory, sent) - for _, m := range rest { - p.Modules[m].SentAt = &now - p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat) - } - // One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974). - for _, n := range sent { - var carried []string - for _, m := range rest { - if slices.Contains(restTo[m], n) { - carried = append(carried, m) - } - } - noteSends(ctx, p.Tier, []string{n}, carried, now) - noteChanges(p, n, carried, was[n], now) - } - fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "), - strings.Join(sent, ", ")) - return true, nil - } - if len(pending) > 0 { - note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") + - " to report it applied before the rest are sent" - changed := p.State != inventory.PlanRolling || p.Note != note - p.State = inventory.PlanRolling - p.Note = note - return changed, nil - } - // And wait for what the next tier needs running. - leftAsWas := "" - needed := gates(*p, edges, rollsOut) - if len(needed) > 0 { - reports, err := inv.LastReports(ctx) - if err != nil { - return false, err - } - var needs []gateNeed - for _, m := range needed { - running, err := inv.Running(ctx, m) - if err != nil { - return false, err - } - state := p.Modules[m] - if state == nil { - state = &inventory.PlanModule{} - p.Modules[m] = state - } - if state.State == planDeleted { - continue - } - // A machine passed as away was sent nothing to wait on (ADR 0435 §1, issue 1161). - running = runningHeard(*p, running, state) - // The plan sends what it waits for. A rebuild from the same source commit is not a - // move the catalogue announces — the build machine rebuilt for a controller change - // is one — so the roll-out that opens this gate is the plan's to make, once, and - // the reports that open it are the ones after the send. - since := latest - if state.BuiltAt != nil { - since = *state.BuiltAt - } - // **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648). - // Nothing was sent for it, so no report follows its marking or its build: a machine that does not - // happen to report again held the walk for ever. Each machine runs it once it answered its last - // send — the one whose builds made it no move — whenever that was. - if noMove(state) { - byNode := map[string]inventory.Reported{} - for _, r := range reports { - byNode[r.Node] = r - } - needs = append(needs, gateNeed{m, running, func(node string) time.Time { - if s := byNode[node].Sent; s != nil { - return *s - } - return time.Time{} - }}) - continue - } - // **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218) - // a module is sent twice — the first machine, then the rest — and SentAt is the second. - // Asked of every machine, the first machine's report, made between the two sends, read as - // older than the build, and the gate waited for a report it already had, for ever. - sinceFor := func(node string) time.Time { - at := since - sent := state.SentAt - for _, n := range state.First { - if n == node { - sent = state.FirstAt - } - } - if sent != nil && sent.After(at) { - at = *sent - } - return at - } - needs = append(needs, gateNeed{m, running, sinceFor}) - } - waiting, quiet, lost := settleTierGate(needs, reports, silent) - if lost { - // Every machine the next tier waits on is silent: nothing it needs was delivered anywhere, so the plan does - // not go on as if it had been (issue 1160). - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("tier %d built; %s %s: %s", p.Tier, silentGateWords, applySilentAfter, strings.Join(quiet, ", ")) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - if len(quiet) > 0 { - fmt.Printf("%s: tier %d: %s said nothing on its send within %s and is left as it was; the plan goes on without it\n", - p.ID, p.Tier, strings.Join(quiet, ", "), applySilentAfter) - leftAsWas = fmt.Sprintf("%s said nothing on its send within %s and is left as it was", strings.Join(quiet, ", "), - applySilentAfter) - } - if len(waiting) > 0 { - note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied" - changed := p.State != inventory.PlanRolling || p.Note != note - p.State = inventory.PlanRolling - p.Note = note - return changed, nil - } - } - // **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §7): after its last tier, every machine it sent - // has reported on that send, or is silent past ApplySilentAfter; never a health reading. - end := "" - if p.Tier == len(p.Tiers)-1 { - reports, err := inv.LastReports(ctx) - if err != nil { - return false, err - } - waiting, failed, silent := walkReports(*p, reports, time.Now().UTC(), provLookup(ctx)) - if len(waiting) > 0 { - note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send" - changed := p.State != inventory.PlanRolling || p.Note != note - p.State = inventory.PlanRolling - p.Note = note - return changed, nil - } - var said []string - if len(failed) > 0 { - said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", ")) - } - if len(silent) > 0 { - said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", applySilentAfter, - strings.Join(silent, ", "))) - } - said = append(said, recordedSaid(*p)...) - end = strings.Join(said, "; ") - // **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360): - // once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind. - if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) { - carryUserList(ctx, open, p) - } - } - if end == "" { - end = leftAsWas - } - // A machine passed as away is said where the others passed are said (ADR 0435 §1, issue 1161). - if said := awaySaying(*p); said != "" { - end = strings.TrimPrefix(end+"; "+said, "; ") - } - p.Tier++ - p.State = inventory.PlanBuilding - p.Note = end - if p.Tier < len(p.Tiers) { - fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", ")) - } - return true, nil -} - -// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4): -// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter, -// each sorted. Pure. -func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) (waiting, failed, silent []string) { - byNode := map[string]inventory.Reported{} - for _, r := range reports { - byNode[r.Node] = r - } - // The modules of this delivery sent to each machine: a machine is judged on those, not on every module its - // declaration hands to the provisioner (ADR 0362 §1-2, issue 840). - carried := map[string][]string{} - for m, s := range p.Modules { - if s == nil { - continue - } - for node := range s.Rest { - carried[node] = append(carried[node], m) - } - } - // **Each machine on the walk's last send to it** (novox/hq ADR 0531, issue 1886): a send carries the machine's whole declaration (ADR - // 0221), so a report on the last one answers every earlier one, and the bound runs from it. Judged on every send - // it had made, a walk whose tiers were sent hours apart — R4e, stopped at tier 1 and resumed — held tier 0's send of - // 00:23 against tier 1's templates and named its machines silent 35 seconds after tier 1 was sent. - last := map[string]inventory.SentDeclaration{} - lastAt := map[string]time.Time{} - for _, s := range p.Modules { - if s == nil { - continue - } - for node, sent := range s.Rest { - at := sent.At - if at.IsZero() && s.SentAt != nil { - at = *s.SentAt - } - was, ok := last[node] - if !ok || laterSend(sent, at, was, lastAt[node]) { - last[node], lastAt[node] = sent, at - } - } - } - for node, sent := range last { - at := lastAt[node] - sent = sent.Only(carried[node]...) - r := byNode[node] - // A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5). - outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov) - switch { - case reported: - switch { - case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused: - failed = append(failed, node) - case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed): - failed = append(failed, node+" ("+outcome+")") - } - case !at.IsZero() && now.Sub(at) > applySilentAfter: - silent = append(silent, node) - default: - waiting = append(waiting, node) - } - } - sort.Strings(waiting) - sort.Strings(failed) - sort.Strings(silent) - return waiting, failed, silent -} - -// laterSend is whether a, sent at aAt, is a later send to a machine than b, sent at bAt: by its sequence when both carry -// one, else by when it went. -func laterSend(a inventory.SentDeclaration, aAt time.Time, b inventory.SentDeclaration, bAt time.Time) bool { - if a.Sequence > 0 && b.Sequence > 0 && a.Sequence != b.Sequence { - return a.Sequence > b.Sequence - } - return aAt.After(bAt) -} - -// restOf is, for one module, every machine of its rest that the send reached and what it carried there. -func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration { - out := map[string]inventory.SentDeclaration{} - for _, n := range to { - if slices.Contains(sent, n) { - out[n] = what[n] - } - } - if len(out) == 0 { - return nil - } - return out -} - -// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is -// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once, -// before the send, so a module the same send carries is never read as already moved — the machines it -// went to, and the gate. The gate is kept on the first of them and judges everything the send moved; -// the others are gated by it. -func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, - runningOf map[string][]string) error { - inv := open.inventory - before, beforeKnown, _ := inv.SentBuilds(ctx, node) - // **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan - // before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from - // the readings of a gate already judging it there. - // Only a send the machine reported it applied is one it has; one it never reported on is sent, as before - // (issue 280: what it ran before is then that build, and a failure leaves it as it was). - if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 && - appliedLastSend(ctx, inv, node) { - if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 { - adoptSend(ctx, open, p, node, modules, before, runningOf) - return nil - } - } - // **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced - // between the send and keeping it reads it here, and a failure still puts the machine back. - kept := false - for _, m := range modules { - s := p.Modules[m] - if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] { - s.Previous, kept = before[m], true - } - } - if kept { - if err := inv.SavePlan(ctx, p); err != nil { - return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err) - } - } - owns := make([]inventory.CarriedMove, 0, len(modules)) - for _, m := range modules { - s := p.Modules[m] - owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build}) - } - // What the assignments name before the send, for the rollout to say what it changed; and the send is made as this - // rollout, so what is recorded of it names it (novox/hq ADR 0408, issue 1095). - was := ranBefore(ctx, inv, node) - ctx = sendingAs(ctx, p) - // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges - // all of it there. - carried, sent, grants, err := gatedSend(ctx, open, node, owns) - if err != nil { - return err - } - now := time.Now().UTC() - lead := modules[0] - // What each machine was just sent, kept on the gate (novox/hq issue 352): its report is held against - // this send, whatever it is sent after. - sentWhat := sentNow(ctx, inv, sent) - for _, m := range modules { - s := p.Modules[m] - // What the first machine ran before (ADR 0236), kept above before the send. - s.First, s.FirstAt = sent, &now - s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]), - From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat} - s.GatedBy = "" - if m == lead { - s.Gate.Carried = carried - s.Gate.Grants = grants - } else { - s.GatedBy = lead - } - } - noteSends(ctx, p.Tier, sent, modules, now) - for _, n := range sent { - noteChanges(p, n, modules, was, now) - } - p.State = inventory.PlanRolling - p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "), - strings.Join(sent, ", ")) - fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n", - p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", ")) - // The grants step taken before it (novox/hq issue 490), said with it. - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - // What the send recreates, said with it (novox/hq ADR 0245). - if said := recreationsSaid(carried); said != "" { - p.Note += "; " + said - fmt.Printf("%s: %s\n", p.ID, said) - } - return nil -} - -// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts -// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build -// there — whose carried move also says what the machine ran before, for a rollback. -func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, - before map[string]string, runningOf map[string][]string) { - inv := open.inventory - since := time.Now().UTC() - last, found, err := inv.SentTo(ctx, node) - if err == nil && found && !last.At.IsZero() { - since = last.At - } - lead := modules[0] - other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build) - for _, m := range modules { - s := p.Modules[m] - // What it ran before: kept by this walk before its send was made (§6); else what the other walk's send - // moved it from; else the build itself, so a failure leaves it as it was (issue 280). - switch { - case s.Previous != "": - case m == lead && from != "": - s.Previous = from - default: - s.Previous = before[m] - } - at := since - s.First, s.FirstAt = []string{node}, &at - s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]), - From: s.Previous, To: s.Commit, Since: &at} - if found { - s.Gate.Sent = map[string]inventory.SentDeclaration{node: last} - } - s.GatedBy = "" - if m == lead { - s.Gate.TakeReadings(other, modules, id) - } else { - s.GatedBy = lead - } - } - p.State = inventory.PlanRolling - p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send", - p.Tier, node, strings.Join(modules, ", ")) - if id != "" { - p.Note += ", going on from the readings of " + id - } - fmt.Printf("%s: %s\n", p.ID, p.Note) -} - -// appliedLastSend says a machine reported it applied the declaration it was sent last. -func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool { - reports, err := inv.LastReports(ctx) - if err != nil { - return false - } - for _, r := range reports { - if r.Node == node { - return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied - } - } - return false -} - -// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the -// walk's id, and what the machine ran before; nil when none does. -func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, "", "" - } - // The same build, by its id where both name one, else by its commit. - same := func(b, c string) bool { - if b != "" && build != "" { - return b == build - } - return sameCommit(c, commit) - } - usable := func(g *inventory.PlanGate) bool { - return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) && - slices.Contains(g.Machines, node) - } - for _, q := range plans { - if q.ID == self { - continue - } - if r := q.Release; r != nil { - gates := []*inventory.PlanGate{r.Gate} - for _, g := range r.Gates { - gates = append(gates, g) - } - for _, g := range gates { - if !usable(g) { - continue - } - for _, c := range g.Carried { - if c.Module == module && c.Node == node && same(c.Build, c.To) { - return g, q.ID, c.From - } - } - } - continue - } - if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) { - return s.Gate, q.ID, s.Previous - } - } - return nil, "", "" -} - -// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate -// on the first of them passed, and its pass was kept for every build it carried (passCarried). -func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) { - if s.Gate == nil { - s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since} - } - if s.Gate.Verdict != "" { - return - } - s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took - s.Gate.Passes, s.Gate.Kept = g.Passes, true -} - -// failFirstSend stops the plan at a first send whose gate failed, or whose machine refused or failed -// what it was sent: what the gate found wanting is put back there, each once — the module the gate is -// kept on only when it is among them (issue 281: the module whose gate it is was blamed for whatever -// the send carried) — and the machines after it are left as they were. -func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, - machines []string, why string, rest []string) { - batched, back := batchingRollbacks(ctx) - defer func() { - sendRollbacks(ctx, open, p, back) - p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(rest, ", "))) - fmt.Printf("%s: %s\n", p.ID, p.Note) - }() - g := state.Gate - if g != nil && g.Verdict == inventory.GatePassed { - // **A guard: a build that passed its gate is never put back for what came after** (novox/hq issue 335). - // Not reached while nextRollout answers a passed gate with the rest to send, and advanceOnce reads it - // after a carried module takes over its lead's pass; it is here so that a path added later cannot - // overturn a verdict. If it is reached, the plan stops and says why, and the build is not marked failed. - // The module is left a stopped rollout (its Why said, sent first and not to the rest), which - // `plans retry` takes: it sends the first machine again, and the passed gate then sends the rest. - state.Why = "passed its gate; its walk then stopped: " + why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s passed its gate on %s in tier %d (%s) and is kept; its walk stopped after: %s", - module, strings.Join(g.Machines, ", "), p.Tier, g.Why, why) - return - } - if g != nil && slices.Contains(g.Returned, module) { - // Put back at once when it broke: its rollback was made then, and is not made again. - state.Why = "put back when it broke; its send failed: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was put back when it broke", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module) - } else if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) { - gateFailed(batched, open, p, module, state, machines, why) - } else if slices.Contains(g.Passing, module) && state.Build != "" { - // **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes - // asked while another of its send failed, its build has a verdict, and other walks do not wait on it. - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, - Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed, - Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil { - fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err) - } - state.Why = "passed on its own; stopped with the send that carried it: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module) - } else { - state.Why = "not found wanting; stopped with the send that carried it: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was not found wanting and is left as it is", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module) - } - if state.Gate != nil && len(state.Gate.Carried) > 0 { - failCarried(batched, open, p, state.Gate, module) - } -} - -// rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249). -type rolloutStep struct { - // send is the machines to send now; first, whether they are the first machine's send. - send []string - first bool - // waiting names the first machines whose report the rest wait for. - waiting string - // failed says how a first machine did not take it; rest is what is then left alone. - failed string - rest []string -} - -// nextRollout is the next step of one module's rollout in a plan (novox/hq issue 249, ADR 0218). -// -// Together, every machine running it at once, as the policy says. Otherwise one machine first — the -// first by name among those that have reported within the bound, so a laptop that is away is not -// the one the rest wait on; the first by name when none has; the same choice on every controller and -// every resume — and the rest once each machine the first send reached reports, about the -// declaration it was last sent, that it applied it. Compared by the store's own record of what was -// sent (Reported.Current), never by this controller's clock against the machine's. -// -// **A first machine that fails, refuses, or does not report within the bound stops the rollout -// there** (ADR 0218 §2), naming the machine; the rest are not sent. A wait with no end is not a -// rollout: it held the plan open for ever, read as work in progress (issue 254). -func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported, - now time.Time, bound time.Duration) rolloutStep { - if len(running) == 0 { - return rolloutStep{} - } - if together { - return rolloutStep{send: running} - } - byNode := map[string]inventory.Reported{} - for _, r := range reports { - byNode[r.Node] = r - } - if s.FirstAt == nil { - sorted := append([]string{}, running...) - sort.Strings(sorted) - for _, n := range sorted { - if r, said := byNode[n]; said && r.At != nil && now.Sub(*r.At) <= bound { - return rolloutStep{send: []string{n}, first: true} - } - } - return rolloutStep{send: sorted[:1], first: true} - } - sentFirst := map[string]bool{} - for _, n := range s.First { - sentFirst[n] = true - } - var rest []string - for _, n := range running { - if !sentFirst[n] { - rest = append(rest, n) - } - } - // **A passed gate is the first machine's verdict, and its later reports are not** (novox/hq issue 335). - // Once the build passed there, what that machine reports next is about whatever it was sent after — - // another walk's send, a push — and says nothing of this build. On 2026-10-08 a build passed on the - // laptop, its send to the rest waited on another walk, that walk sent the laptop a new declaration it did - // not report for half an hour, and the plan read the silence as the first machine never applying the - // passed build: it marked it failed at its gate and put it back. The rest are sent, as the pass said. - if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed { - return rolloutStep{send: rest} - } - var waiting, failed []string - for _, n := range s.First { - r, said := byNode[n] - // Only a report about what this plan sent it — or what it was sent after that — says anything about - // this build (novox/hq issue 352); a plan from before sends were kept on the gate reads the report - // against the send made last, as before. - reported := r.Current - if s.Gate != nil && s.Gate.Sent != nil { - sent, kept := s.Gate.Sent[n] - reported = kept && sent.ReportsOn(r) - } - if !said || r.At == nil || !reported { - waiting = append(waiting, n) - continue - } - switch r.Outcome { - case inventory.OutcomeApplied: - case inventory.OutcomeFailed, inventory.OutcomeRefused: - failed = append(failed, n+" "+r.Outcome+" what it was sent") - default: - waiting = append(waiting, n) - } - } - if len(failed) > 0 { - return rolloutStep{failed: strings.Join(failed, "; "), rest: rest} - } - if len(waiting) > 0 { - if now.Sub(*s.FirstAt) > bound { - return rolloutStep{failed: fmt.Sprintf("%s did not report it applied within %s", - strings.Join(waiting, ", "), bound), rest: rest} - } - return rolloutStep{waiting: strings.Join(waiting, ", ")} - } - return rolloutStep{send: rest} -} - -// sayUnsent adds to an ended plan's note the modules it built and never sent (novox/hq issue 249). -// An announced move of a module a plan held was left to that plan; a plan that ends without -// sending it — failed elsewhere, or closed by hand — would leave its machines behind with nothing -// saying so. A module whose rollout stopped at its first machine is not among them: that stop was -// the point. Said once. -func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) { - var unsent []string - for name, s := range p.Modules { - if s != nil && s.State == "built" && s.SentAt == nil && s.FirstAt == nil && rollsOut(name) { - unsent = append(unsent, name) - } - } - if len(unsent) == 0 || strings.Contains(p.Note, "built and never sent") { - return - } - sort.Strings(unsent) - p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `upgrade release-backlog --why ` releases them" -} - -// planLine is one plan as `status` says it, given the least bound of a tier. -func planLine(p inventory.Plan, now time.Time) string { - return planLineWith(p, now, tierAtLeast) -} - -// inTierSince is when a plan began waiting on what it waits on now, which `plans`, `status` and the -// watchdog of a plan's progress (S3) all count from (novox/hq issue 296): a walk waiting for its -// delivery's word from when it was opened, as S16 counts; any other plan from when it entered its tier, -// or, when the delivery let it go after that, from the go. Never from the plan's last save: a plan is -// saved for many reasons while its tier stands still, and a clock reset by each save said a build queued -// for minutes had been building for a few seconds. A plan read without the stamp (one from before it -// was kept) counts from its last save, the only time it has. -func inTierSince(p inventory.Plan) time.Time { - if p.Waiting() { - return p.Created - } - since := p.TierEntered - if since.IsZero() { - since = p.Updated - } - if d := p.Delivery; d != nil && d.Go != nil && d.Go.After(since) { - since = *d.Go - } - return since -} - -// planLineWith is planLine with bound, the plan's tier bound: bound is -// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition -// said on the line (novox/hq issue 296). The machines running what it moves are not named: planLineOn. -func planLineWith(p inventory.Plan, now time.Time, bound time.Duration) string { - return planLineOn(p, now, bound, nil) -} - -// planLineOn is planLineWith naming the plan by what it moves and where (planHeadline), given what runs each -// module; nil names no machine. -func planLineOn(p inventory.Plan, now time.Time, bound time.Duration, running func(string) []string) string { - return planLineSaid(p, now, bound, running, nil) -} - -// planLineSaid is planLineOn saying, of a walk waiting for its delivery's word, what the delivery's owner says -// holds it (novox/hq issue 529); nil waits says it as before. -func planLineSaid(p inventory.Plan, now time.Time, bound time.Duration, running func(string) []string, - waits walkWaits) string { - name := planHeadline(p, running) - where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers)) - switch p.State { - case inventory.PlanAssembling, inventory.PlanQueued: - // A batch not yet a walk (novox/hq ADR 0276): what it holds and how long is left. - return batchWords(p, now) - case inventory.PlanDone: - return fmt.Sprintf("%s · done, %d tier(s)", name, len(p.Tiers)) - case inventory.PlanFailed: - return fmt.Sprintf("%s · FAILED at %s: %s", name, where, p.Note) - case inventory.PlanSuperseded: - return fmt.Sprintf("%s · %s", name, p.Note) - } - since := now.Sub(inTierSince(p)).Round(time.Second) - if p.Waiting() { - // Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239). - return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingSaid(p, waits[p.ID]), since) - } - if deferred(p) { - // Nor is waiting for the walk before it to end (a related walk, novox/hq ADR 0276, ADR 0407). - return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) - } - if waitingOnBackups(p) { - return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) - } - late := "" - if since > bound { - late = " — LATE" - } - what := "building" - if p.State == inventory.PlanRolling { - what = p.Note - } - if p.State == inventory.PlanBuilding && refused(p) && currentTierBuilt(p) { - // Every build of the tier is in, so the plan waits on something else, and its note says what: a send - // refused ("a build waits there for its gate: (...)") is kept there. "building" with LATE - // hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920). - what = p.Note - // And when the builds are held after a failed release, they wait for a person, not for a build, and the - // operator is asked to release them (novox/hq issue 1732): said so, and not late. - if strings.Contains(p.Note, errUngated.Error()) && backlogNow.held != "" { - what += " — held after a failed delivery until a person releases it (the operator is asked)" - late = "" - } - return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late) - } - return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late) -} - -// planHeadline names a plan as a person knows it (asked by the operator, 2026-10-10: a plan called by its -// repository alone said nothing of what it delivers): each repository as its pull request and title, what the -// plan moves, and the machines running that — "mesh-catalog #175 a tap shows its outcome · messenger, -// telegram → novox". A record naming no pull request is named by its commit, as before; one naming no moves -// says none; running nil names no machine. -func planHeadline(p inventory.Plan, running func(string) []string) string { - var repos []string - moves := map[string]bool{} - for _, c := range p.Carried() { - seg := repoName(c.Repository) + " " + short(c.Commit) - if p.Delivery != nil { - for _, m := range p.Delivery.Merges { - if !strings.EqualFold(m.Repository, c.Repository) { - continue - } - for _, n := range m.Moves { - moves[n] = true - } - if m.Commit == c.Commit && m.Number > 0 { - seg = repoName(c.Repository) + " " + pullWords(m) - } - } - } - repos = append(repos, seg) - } - out := strings.Join(repos, " + ") - if len(moves) == 0 { - return out - } - names := sortedKeysOf(boolsToStrings(moves)) - out += " · " + strings.Join(names, ", ") - if running == nil { - return out - } - nodes := map[string]bool{} - for _, n := range names { - for _, node := range running(n) { - nodes[node] = true - } - } - if len(nodes) > 0 { - out += " → " + strings.Join(sortedKeysOf(boolsToStrings(nodes)), ", ") - } - return out -} - -func repositoryMatches(a, b string) bool { - trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) } - return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a)) -} - -// planStatus is one plan as `status --json` says it. -type planStatus struct { - ID string `json:"id"` - Repository string `json:"repository"` - Commit string `json:"commit"` - State string `json:"state"` - Tier int `json:"tier"` - Tiers int `json:"tiers"` - Waiting string `json:"waiting,omitempty"` - Since time.Time `json:"since"` - Late bool `json:"late"` -} - -func planStatuses(plans []inventory.Plan, now time.Time, bounds tierBounds) []planStatus { - out := make([]planStatus, 0, len(plans)) - for _, p := range plans { - ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State, - Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated} - if p.Open() { - ps.Since = inTierSince(p) - ps.Waiting = p.Note - if ps.Waiting == "" { - ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier) - } - // A walk waiting for its delivery's word is no tier late (ADR 0239): S16 is its watchdog. - ps.Late = !p.Waiting() && now.Sub(ps.Since) > bounds.of(p.Repository) - } - out = append(out, ps) - } - return out -} - -// openPlans is the open plans among the recent ones, and how many have waited past the bound. -func openPlans(plans []inventory.Plan, now time.Time, bounds tierBounds) ([]inventory.Plan, int) { - var open []inventory.Plan - late := 0 - for _, p := range plans { - if p.Open() { - open = append(open, p) - if p.Waiting() { - continue - } - if now.Sub(inTierSince(p)) > bounds.of(p.Repository) { - late++ - } - } - } - return open, late -} - -// plansCommand says what the last merges produced and where each stands; given an id, one plan -// tier by tier with every module's state. -func plansCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("plans", flag.ContinueOnError) - limit := set.Int("n", 10, "how many to show") - whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules") - paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root") - modules := set.String("modules", "", "or the modules it would change, comma-separated") - moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+ - "comma-separated, as the forge's announcer says them (novox/hq issue 278)") - // The what-if as data, the document `delivery plan` answers (novox/hq ADR 0450 §2, issue 1324). - data := set.Bool("data", false, "with --what-if: answer the change plan as data, the document `delivery plan` answers; "+ - "without it the answer is the text a person reads") - head := set.String("head", "", "with --data: the commit at hand") - base := set.String("base", "main", "with --data: the branch it merges into") - removed := set.String("removed", "", "with --data: the files among --paths it deletes, comma-separated") - // Following a plan (novox/hq issue 229): `plans wait `. - until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)") - bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+ - "longest a call is answered within)") - // Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7). - why := addHandActFlags(set) - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close" || positionals[0] == "go") { - // Refused before anything is opened: a repair by hand says why. - if err := why.require("plans " + positionals[0]); err != nil { - return err - } - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - now := time.Now() - if len(positionals) == 2 && positionals[0] == "wait" { - return planWaitCommand(ctx, inv, positionals[1], *until, *bound) - } - if len(positionals) == 1 { - p, err := inv.PlanByID(ctx, positionals[0]) - if err != nil { - return err - } - bounds := readTierBounds(ctx, inv, now) - fmt.Printf("%s — %s\n", p.ID, planLineSaid(p, now, - bounds.of(p.Repository), func(module string) []string { - on, _ := inv.Running(ctx, module) - return on - }, waitsOf(ctx, inv, []inventory.Plan{p}))+heldSaid(p, heldNow(ctx, inv))) - if r := p.Release; r != nil { - // A release plan's walk (ADR 0236): machines done, the one judged, those to come. - fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), - orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", "))) - if len(r.Failed)+len(r.Silent) > 0 { - fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", "))) - } - gates := []*inventory.PlanGate{r.Gate} - for _, n := range r.Order { - gates = append(gates, r.Gates[n]) - } - for _, g := range gates { - if g == nil { - continue - } - fmt.Printf(" %s\n", gateLine(g)) - for _, c := range g.Carried { - fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To)) - if c.Recreates != "" { - fmt.Printf(" %-22s %s\n", "", c.Recreates) - } - } - } - return nil - } - for i, tier := range p.Tiers { - marker := " " - if i == p.Tier && p.Open() { - marker = ">" - } - fmt.Printf("%s tier %d\n", marker, i) - for _, m := range tier { - s := p.Modules[m] - state := "not yet asked" - if s != nil && s.State != "" { - state = s.State - if s.Commit != "" { - state += " from " + short(s.Commit) - } - if s.Build != "" && s.State != "built" { - state += " (" + s.Build + ")" - } - if s.Why != "" { - state += ": " + s.Why - } - } - fmt.Printf(" %-22s %s\n", m, state) - // The rollout's record at its gate (novox/hq to-be 45 §8): first machine, from and to, - // verdict, time to it, rolled back or not. - switch { - case s != nil && s.GatedBy != "" && (s.Gate == nil || s.Gate.Verdict == ""): - // Sent with others, judged by the gate of that send (issue 281). - fmt.Printf(" %-22s judged with %s, in the send to %s\n", "", s.GatedBy, strings.Join(s.First, ", ")) - case s != nil && s.Gate != nil: - fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate)) - } - // Which way a recorded module went on each machine (novox/hq ADR 0377 §6). - if s != nil { - for _, n := range sortedKeysOf(s.Recorded) { - fmt.Printf(" %-22s on %s %s\n", "", n, s.Recorded[n]) - } - } - // What the send to its first machine did to its containers (ADR 0245). - if s != nil && s.Gate != nil { - for _, c := range s.Gate.Carried { - if c.Recreates != "" { - fmt.Printf(" %-22s on %s %s\n", "", c.Node, c.Recreates) - } - } - } - } - } - // The assignments it changed, each with the build and commit before and after (novox/hq ADR 0408, issue 1095). - for _, line := range changedLines(p) { - fmt.Println(line) - } - return nil - } - if *data && *whatIf == "" { - return errors.New("--data answers a what-if: give --what-if owner/repository") - } - if *whatIf != "" && *data { - if *paths == "" { - return errors.New("plans --what-if --data: the change plan is of the files a change touches; give --paths") - } - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - return err - } - m := changeMember(*whatIf, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs)) - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - return enc.Encode(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) { - u, err := inv.UpgradeOf(ctx, module) - return u, err == nil - })) - } - if *whatIf != "" { - return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules), moduleDirsOf(*moduleDirs)) - } - // `go` (novox/hq ADR 0239): a person's word in place of the delivery's owner, for a walk that waits for it - // — mesh-delivery down, or a person who will not wait. Recorded in the hand-act log with why. - if len(positionals) == 2 && positionals[0] == "go" { - why.record(ctx, "plans go", positionals[1:]) - var p inventory.Plan - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, err = letGo(ctx, inv, positionals[1], "a person ("+whoAsked()+")", strings.TrimSpace(*why.why)) - return err - }); err != nil { - return err - } - fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository, - short(p.Commit)) - return nil - } - // `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on. - if len(positionals) == 2 && positionals[0] == "retry" { - // Said on the bus (novox/hq ADR 0308 decision 1): a retried walk starts again, and its start is said - // by the verb that starts it. - var said string - if err := sayingOnTheBus(ctx, inv, func() (err error) { - said, err = retryPlan(ctx, open, positionals[1]) - return err - }); err != nil { - return err - } - fmt.Println(said) - return nil - } - // `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one - // waiting on a report that cannot come — so it stops reading as work in progress. Marked failed - // with who ended it; what it asked still builds and registers. - if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") { - how := "stopped" - if positionals[0] == "close" { - how = "closed" - } - // Said on the bus (novox/hq ADR 0308 decision 1): the walk's end, outcome stopped, is said by the verb - // that ends it, so what it touched is not held until its bound. - var p inventory.Plan - if err := sayingOnTheBus(ctx, inv, func() error { - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return err - } - defer release() - if p, err = inv.PlanByID(ctx, positionals[1]); err != nil { - return err - } - if !p.Open() { - return fmt.Errorf("%s is already %s", p.ID, p.State) - } - why.record(ctx, "plans "+positionals[0], positionals[1:]) - stopByHand(&p, how, "a person ("+whoAsked()+")", strings.TrimSpace(*why.why)) - sayUnsent(&p, func(m string) bool { - u, err := inv.UpgradeOf(ctx, m) - return err == nil && u.RollOut - }) - return inv.SavePlan(ctx, &p) - }); err != nil { - return err - } - fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n", - p.ID, how, p.Tier, len(p.Tiers)) - return nil - } - plans, err := inv.RecentPlans(ctx, *limit) - if err != nil { - return err - } - // **The batch being assembled first** (novox/hq ADR 0276 decision 7): what it holds, how long is left, - // and that its plan is not yet calculated. - batches, err := inv.Batches(ctx) - if err != nil { - return err - } - if len(plans) == 0 && len(batches) == 0 { - fmt.Println("no merge has produced a plan yet") - return nil - } - for _, b := range batches { - fmt.Printf("%-28s %s\n", b.ID, batchWords(b, now)) - // One line per repository: its pull request, what it answers, what it moves. - for _, line := range batchLines(b) { - fmt.Printf("%-28s %s\n", "", line) - } - } - bounds := readTierBounds(ctx, inv, now) - running := func(module string) []string { - on, _ := inv.Running(ctx, module) - return on - } - waits := waitsOf(ctx, inv, plans) - held := heldNow(ctx, inv) - for _, p := range plans { - if p.Batch() { - continue - } - fmt.Printf("%-28s %s%s\n", p.ID, planLineSaid(p, now, bounds.of(p.Repository), running, waits), heldSaid(p, held)) - } - return nil -} - -// waitsOf asks the delivery's owner what holds the walks among these that wait for its word (novox/hq issue 529), -// once, and only when one waits; nil when none waits, no holder is on record, or the bus or the owner does not -// answer within a few seconds — the lines then read as before. -func waitsOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) walkWaits { - waiting := false - for _, p := range plans { - waiting = waiting || p.Open() && p.Waiting() - } - if !waiting { - return nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil - } - ctx, cancel := context.WithTimeout(ctx, 3*time.Second) - defer cancel() - var waits walkWaits - _ = onTheBus(func(conn *nats.Conn) error { - waits = walksWaiting(ctx, conn, deliverySeatHeld(entries)) - return nil - }) - return waits -} - -// planWhatIf is the plan a merge would produce, computed the way the merge handler computes one -// and saved nowhere: the modules the repository's changed files touch (or the modules named), what -// packages their source, everything reachable from them, in tiers. For reading before merging. -func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string, paths, modules []string, moduleDirs *[]string) error { - owner, repo, found := strings.Cut(repository, "/") - if !found { - return fmt.Errorf("--what-if takes owner/repository, not %q", repository) - } - m := link.SourceMoved{Owner: owner, Repo: repo, Base: "main", Commit: "what-if", Paths: paths} - if moduleDirs != nil { - m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return err - } - var from, packaging []inventory.Entry - deleted := map[string]bool{} - named := map[string]bool{} - for _, name := range modules { - named[name] = true - } - if len(named) == 0 { - // The changed files: the planner's own answer, as the merge handler, the merge gate and a pull - // request's check ask it (novox/hq ADR 0238). - r := reachOfMerge(m, entries, read, nil) - from, packaging = append(append([]inventory.Entry{}, r.Touched...), r.Deleted...), r.Packaging - for _, e := range r.Deleted { - deleted[e.Manifest.Module] = true - } - } else { - for _, e := range entries { - switch { - case named[e.Manifest.Module]: - from = append(from, e) - case readsFrom(read[e.Manifest.Module], m): - packaging = append(packaging, e) - } - } - } - moved := append(append([]inventory.Entry{}, from...), packaging...) - if len(moved) == 0 { - fmt.Printf("a merge of %s changing %s would build nothing the mesh holds\n", repository, - orNone(strings.Join(append(paths, modules...), ", "))) - return nil - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return err - } - var names []string - for _, e := range moved { - names = append(names, e.Manifest.Module) - } - p := planOfMerge(m, names, edges) - fmt.Printf("a merge of %s would build %d module(s) in %d tier(s):\n", repository, len(p.Modules), len(p.Tiers)) - why := map[string]string{} - for _, e := range packaging { - why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m) - } - if len(named) == 0 { - for _, e := range from { - why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m) - if deleted[e.Manifest.Module] { - why[e.Manifest.Module] = "its manifest is removed: deleted at its source, forgotten where nothing holds it, not built" - } - } - } - rolls := map[string]string{} - for i, tier := range p.Tiers { - fmt.Printf(" tier %d\n", i) - for _, name := range tier { - how := "built; its policy records, so nothing is sent" - if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut { - running, _ := inv.Running(ctx, name) - reports, _ := inv.LastReports(ctx) - how = "built, then sent to " + orNone(strings.Join(running, ", ")) - // One machine first unless the policy says together (novox/hq issue 249). - if first := nextRollout(inventory.PlanModule{}, running, u.Together, reports, time.Now(), planWaitBound); first.first && len(running) > 1 { - how = fmt.Sprintf("built, then sent to %s first and to the rest once it has applied it", - first.send[0]) - } - rolls[name] = how - } - fmt.Printf(" %-22s %s\n", name, how) - reason := why[name] - switch { - case reason == "" && len(named) > 0 && named[name]: - reason = "named" - case reason == "": - reason = "it stands on a module the merge moves" - } - fmt.Printf(" %-22s why: %s\n", "", reason) - } - } - if hasCycle(p.Tiers, edges) { - fmt.Println(" the last tier depends on itself and would be built together, in no order") - } - return nil -} - -// moduleDirsOf is --module-dirs as an announcer would say it: nothing when not given, so the what-if -// reads as an announcement from an announcer that does not say. -func moduleDirsOf(s string) *[]string { - if strings.TrimSpace(s) == "" { - return nil - } - dirs := splitList(s) - return &dirs -} - -func splitList(s string) []string { - var out []string - for _, part := range strings.Split(s, ",") { - if part = strings.TrimSpace(part); part != "" { - out = append(out, part) - } - } - return out -} - -// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last -// builds and the record of its own ask, by id. -func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) ( - map[string][]inventory.Build, map[string]inventory.Build, error) { - recorded := map[string][]inventory.Build{} - byID := map[string]inventory.Build{} - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.State == "asked" { - builds, err := inv.Builds(ctx, m, 5) - if err != nil { - return nil, nil, err - } - recorded[m] = builds - // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). - if s.Build != "" { - b, found, err := inv.BuildByID(ctx, s.Build) - if err != nil { - return nil, nil, err - } - if found { - byID[s.Build] = b - } - } - } - } - refused, err := refusalsOf(ctx, inv, recorded, byID) - if err != nil { - return nil, nil, err - } - overlayRefusals(recorded, byID, refused) - return recorded, byID, nil -} - -// overlayRefusals marks failed every build record whose registration the mesh refused (novox/hq issue 1708): the record -// keeps the builder's words, which worked, and settled from it as it is the refused build would be built and sent. -func overlayRefusals(recorded map[string][]inventory.Build, byID map[string]inventory.Build, refused map[string]string) { - if len(refused) == 0 { - return - } - fail := func(b *inventory.Build) { - if why, ok := refused[b.ID]; ok && b.Worked() { - b.Failed = why - } - } - for m := range recorded { - for i := range recorded[m] { - fail(&recorded[m][i]) - } - } - for id, b := range byID { - fail(&b) - byID[id] = b - } -} - -// refusalsOf is every refusal kept for the builds these records name, in the store and in this process. -func refusalsOf(ctx context.Context, inv *inventory.Inventory, recorded map[string][]inventory.Build, - byID map[string]inventory.Build) (map[string]string, error) { - var ids []string - for _, bs := range recorded { - for _, b := range bs { - ids = append(ids, b.ID) - } - } - for id := range byID { - ids = append(ids, id) - } - refused, err := inv.RefusedOf(ctx, ids) - if err != nil { - return nil, fmt.Errorf("the refused registrations cannot be read: %w", err) - } - for id, why := range refusalsKeptFor(ids) { - refused[id] = why - } - return refused, nil -} - -// settledEffects is what settling from the records did that needs more than the plan: the modules it found deleted -// at their source, and whether it failed the plan. -func settledEffects(before map[string]string, p *inventory.Plan, was string) (deleted []string, failed bool) { - for m, s := range p.Modules { - if s != nil && s.State == planDeleted && before[m] != planDeleted { - deleted = append(deleted, m) - } - } - sort.Strings(deleted) - return deleted, p.State == inventory.PlanFailed && was != inventory.PlanFailed -} - -// statesOf is each module's state in a plan, for settledEffects. -func statesOf(p *inventory.Plan) map[string]string { - out := map[string]string{} - for m, s := range p.Modules { - if s != nil { - out[m] = s.State - } - } - return out -} - -// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build -// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214). -// Newest first, as Builds answers: the first record after the ask is the outcome of that ask. -// -// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR -// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is -// found by nothing else. -func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build, - byID map[string]inventory.Build) bool { - changed := false - for _, m := range tier { - s := p.Modules[m] - if s == nil || s.State != "asked" || s.AskedAt == nil || s.Key { - continue - } - var outcome *inventory.Build - for i := range recorded[m] { - // Asked under an id, only that id's record answers (ADR 0219), and it is looked up below. - if s.Build != "" { - break - } - b := recorded[m][i] - if b.At.Before(*s.AskedAt) { - break - } - // Recorded after the ask and asked before it: an earlier ask's late outcome, not this - // one's (novox/hq 04-ISSUES/219). - if askedBefore(b.Asked, s.AskedAt) { - continue - } - outcome = &b - } - if own, found := byID[s.Build]; s.Build != "" && found { - outcome = &own - } - if outcome == nil { - continue - } - at := outcome.At - if !outcome.Worked() && deletedAtSource(outcome.Failed) { - s.State, s.Why = planDeleted, "deleted at its source: "+firstLine(outcome.Failed) - fmt.Printf("%s: %s was deleted at its source (%s): not built, and the plan goes on\n", p.ID, m, outcome.ID) - changed = true - continue - } - if outcome.Worked() { - s.State = "built" - s.BuiltAt = &at - s.Commit = outcome.Commit - // What the build did, when its outcome was left to the records with words (issue 1708). - if made := takeMadeOf(outcome.ID); made != "" { - s.Made = made - } - } else { - s.State = "failed" - s.Why = outcome.Failed - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed to build in tier %d", m, p.Tier) - } - fmt.Printf("%s: %s settled from the build records as %s (%s)\n", p.ID, m, s.State, outcome.ID) - changed = true - } - return changed -} - -// askedBefore is whether a build asked at asked was asked before a plan asked for its module — and -// so is not that plan's outcome (novox/hq 04-ISSUES/219). False when either time is not known. -func askedBefore(asked time.Time, planAsked *time.Time) bool { - return !asked.IsZero() && planAsked != nil && asked.Before(*planAsked) -} - -// firstRunning is the machines sent first that run the module — not the machine holding the bus, sent -// with them only for its user list. -func firstRunning(first, running []string) []string { - var out []string - for _, n := range first { - if slices.Contains(running, n) { - out = append(out, n) - } - } - if len(out) == 0 { - return first - } - return out -} - -// planDeleted is a plan's module that the merge deleted at its source: not built, not sent, and no -// failure of the plan (novox/hq ADR 0236). -const planDeleted = "deleted" - -// forgetDeleted forgets a module a plan found deleted at its source, where nothing holds it, and says -// it otherwise. -func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string, p *inventory.Plan) { - switch err := inv.ForgetModule(ctx, module); { - case err == nil: - fmt.Printf("%s: %s was deleted at its source: forgotten, and the plan goes on\n", p.ID, module) - default: - fmt.Printf("%s: %s was deleted at its source and is not built; it is kept until a person unassigns it and "+ - "`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error())) - } -} - -// stopByHand ends a walk a person stopped or closed (`plans stop|close`): failed, with who stopped it and why -// kept as the delivery's owner keeps its own stop, so the walk ends with outcome stopped, not failed (novox/hq -// ADR 0308 decision 1). -func stopByHand(p *inventory.Plan, how, by, why string) { - p.State = inventory.PlanFailed - p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + why - if p.Delivery == nil { - p.Delivery = &inventory.PlanDelivery{} - } - p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why -} - -// keyed says a module of the tier still waits on the builder's key. -func keyed(p *inventory.Plan, tier []string) bool { - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.Key && s.State == "asked" { - return true - } - } - return false -} diff --git a/cmd/mesh-controller/release_plan_test.go b/cmd/mesh-controller/release_plan_test.go deleted file mode 100644 index 5425de3c..00000000 --- a/cmd/mesh-controller/release_plan_test.go +++ /dev/null @@ -1,264 +0,0 @@ -package main - -import ( - "slices" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A merge produces a tiered plan (novox/hq ADR 0162): what moved and everything reachable from it, -// sorted so a tier depends only on earlier ones — with the three kinds of dependency told apart. -func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { - edges := []inventory.Edge{ - // build dependencies: images on the runtime, a plugin on one of them - {From: "shop", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - {From: "postgres", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - {From: "shop-plugin", To: "shop", Kind: inventory.EdgeDeclared}, - // a code dependency: the proxy packages the controller's source — same tier - {From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages}, - {From: "builder", To: "mesh-controller", Kind: inventory.EdgePackages}, - // runtime dependencies: everything source-built is built by the builder - {From: "shop", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "postgres", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "shop-plugin", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "route-proxy", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - // the build seat's holder follows the controller that defines its worker (hq issue 206) - {From: "builder", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - {From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn}, - } - // The runtime image moved: everything on it, and what is built by what is on it. - set := reachableFrom([]string{"mesh-tools"}, edges) - // What stands on the runtime, and the builder that stands on it; not the controller, which the - // builder merely builds, nor the proxy that packages the controller. - want := []string{"builder", "mesh-tools", "postgres", "shop", "shop-plugin"} - if len(set) != len(want) { - t.Fatalf("reachable from the runtime: %v, want %v", set, want) - } - tiers := tiersOf(set, edges) - pos := map[string]int{} - for i, tier := range tiers { - for _, m := range tier { - pos[m] = i - } - } - if pos["mesh-tools"] != 0 || pos["builder"] != 1 { - t.Fatalf("the runtime then the builder: %v", tiers) - } - // Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it - // stands on. - if pos["shop"] != 1 || pos["postgres"] != 1 { - t.Fatalf("what the builder builds waits for the builder: %v", tiers) - } - if pos["shop-plugin"] <= pos["shop"] { - t.Fatalf("a plugin after what it is declared on: %v", tiers) - } - if hasCycle(tiers, edges) { - t.Fatalf("no cycle here: %v", tiers) - } - - // The controller alone moved: the controller alone — what packages its repository moves only when the - // change is in its own build source (novox/hq ADR 0267), so a packages edge widens nothing. - if alone := reachableFrom([]string{"mesh-controller"}, edges); len(alone) != 1 { - t.Fatalf("a controller merge rebuilds the controller alone: %v", alone) - } - // A change to a package all three build from moves all three. The builder holds - // the build seat, whose worker the controller defines, so it follows the controller (worker-of, - // novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the - // build machine that is running. The proxy is built by whichever builder runs, so it goes beside the - // controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before). - small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges) - smallTiers := tiersOf(small, edges) - if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" { - t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers) - } - // The builder alone moved: the builder, and nothing it builds. - if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 { - t.Fatalf("a build machine change rebuilds the build machine alone: %v", only) - } - - // Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it. - p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges) - p.Tier = pos["builder"] - rollsOut := func(m string) bool { return m == "builder" } - if g := gates(p, edges, rollsOut); len(g) != 0 { - t.Fatalf("the builder gates the tier after it: %v", g) - } - p.Tier = 0 - if g := gates(p, edges, rollsOut); len(g) != 0 { - t.Fatalf("the runtime image is a build dependency and gates nothing: %v", g) - } - if g := gates(p, edges, func(string) bool { return false }); len(g) != 0 { - t.Fatalf("a module that only records its upgrade gates nothing: %v", g) - } -} - -// A gate is open once every machine running the module has reported after it was built. -func TestAGateOpensWhenTheMachinesHaveReportedSinceTheBuild(t *testing.T) { - built := time.Date(2026, 10, 1, 15, 0, 0, 0, time.UTC) - before, after := built.Add(-time.Minute), built.Add(time.Minute) - reports := []inventory.Reported{{Node: "anchor", At: &after}, {Node: "home-server", At: &before}} - ok, waiting := applied("builder", built, []string{"anchor", "home-server"}, reports) - if ok || len(waiting) != 1 || waiting[0] != "home-server" { - t.Fatalf("one machine has not reported since the build: ok=%v waiting=%v", ok, waiting) - } - if ok, _ := applied("builder", built, []string{"anchor"}, reports); !ok { - t.Fatal("the machine that reported after the build holds the gate open") - } - if ok, _ := applied("builder", built, nil, reports); !ok { - t.Fatal("a module running nowhere gates nothing") - } -} - -// A cycle is not lost: what remains is one last tier, and the caller says so. -func TestACycleIsOneLastTierAndSaidSo(t *testing.T) { - edges := []inventory.Edge{{From: "a", To: "b", Kind: inventory.EdgeStandsOn}, {From: "b", To: "a", Kind: inventory.EdgeStandsOn}} - tiers := tiersOf([]string{"a", "b"}, edges) - if len(tiers) != 1 || len(tiers[0]) != 2 || !hasCycle(tiers, edges) { - t.Fatalf("a cycle should be one tier of two, said: %v", tiers) - } -} - -// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the -// bundle a tier after the toolchain, not beside it. -func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) { - edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}} - p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, - []string{"mesh-tools", "node-tools"}, edges) - if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" { - t.Fatalf("the toolchain, then the bundle: %v", p.Tiers) - } -} - -// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it — -// the controller rebuilding itself — settles from the build records instead of waiting for ever. -func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) { - asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}}, - Modules: map[string]*inventory.PlanModule{ - "mesh-controller": {State: "asked", AskedAt: &asked}, - "builder": {State: "asked", AskedAt: &asked}, - }} - records := map[string][]inventory.Build{ - // Newest first, as Builds answers: the build after the ask is the outcome. - "mesh-controller": { - {ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)}, - {ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)}, - }, - // Only a build from before the ask: not this ask's outcome. - "builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}}, - } - if !settleFromRecords(&p, p.Tiers[0], records, nil) { - t.Fatal("nothing settled, though the controller's build is recorded after the ask") - } - if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil { - t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s) - } - if s := p.Modules["builder"]; s.State != "asked" { - t.Errorf("an ask with no record after it was settled: %+v", s) - } - - // novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier - // plan's late outcome — is not this ask's, built or failed. - r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}}, - Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}} - late := map[string][]inventory.Build{"postgres": { - {ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)}, - }} - if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" { - t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"]) - } - // Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it. - late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73", - Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)}) - if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" || - r.Modules["postgres"].Commit != "4bcd5f73" { - t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"]) - } - - // A failure recorded after the ask fails the plan, as hearing it would have. - q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}}, - Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}} - settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil) - if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" { - t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"]) - } -} - -// The first machine's report, made between the send to it and the send to the rest, opens the gate for it: -// each machine is judged from its own send, not from the last one (novox/hq issue 256). -func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) { - built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC) - firstSent := built.Add(31 * time.Second) - firstReported := built.Add(43 * time.Second) - restSent := built.Add(58 * time.Second) - restReported := built.Add(74 * time.Second) - reports := []inventory.Reported{ - {Node: "ace", At: &firstReported}, - {Node: "g14", At: &restReported}, - } - since := func(node string) time.Time { - if node == "ace" { - return firstSent - } - return restSent - } - if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok { - t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting) - } - // The old reading, every machine from the last send, is what held the plan. - if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok { - t.Fatal("the single-moment reading should hold the first machine back") - } - early := built.Add(10 * time.Second) - if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" { - t.Fatal("a report from before the machine was sent opened the gate") - } -} - -// novox/hq issue 278 and ADR 0236's open question: a change to the build agent — its manifest alone, as -// the catalogue's data sections were, or its program — rebuilds the build agent and nothing it builds. -// What it builds is ordered after it in a plan that holds both, never added to one for its sake. The -// merge that rebuilt 103 modules with the agent in tier 0 rebuilt them for a file read as shared code; -// the agent stood first only because everything else is built by it. -func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) { - const repo = "http://forge.internal:20000/novox/mesh-catalog.git" - agent := fromRepo("build-agent", repo, "modules/build-agent") - redis := fromRepo("redis", repo, "modules/redis") - postgres := fromRepo("postgres", repo, "modules/postgres") - entries := []inventory.Entry{agent, redis, postgres} - edges := []inventory.Edge{ - {From: "redis", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - {From: "redis", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "postgres", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-tools", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - } - for _, paths := range [][]string{ - {"modules/build-agent/module.json"}, // its manifest alone - {"modules/build-agent/cmd/agent/main.go", "modules/build-agent/module.json"}, // its program too - } { - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths, - ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true} - touched := whatTheMergeTouched(entries, entries, m, nil) - if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" { - t.Fatalf("%v touched %v", paths, touched) - } - p := planOfMerge(m, []string{"build-agent"}, edges) - if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "build-agent" || len(p.Modules) != 1 { - t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers) - } - } - // With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing. - p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"}, - []string{"build-agent", "redis"}, edges) - if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 { - t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers) - } -} diff --git a/cmd/mesh-controller/release_provisioned_integration_test.go b/cmd/mesh-controller/release_provisioned_integration_test.go deleted file mode 100644 index 547e0b73..00000000 --- a/cmd/mesh-controller/release_provisioned_integration_test.go +++ /dev/null @@ -1,145 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { - t.Helper() - for i := 0; i < n; i++ { - advancePlans(t.Context(), b.open) - } - return b.release(t) -} - -func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { - judgeLeavesAsTheCore(t) - applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} - - t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) - } - }) - t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || - !strings.Contains(p.Note, "provisioning failed (app: no value)") { - t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) - } - }) - t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil { - t.Fatalf("no gate: %+v", p.Release) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { - t.Fatal(err) - } - p = stepRelease(t, b, 1) - if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("d both applied: the gate passes", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = applied - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { - b := aBacklog(t) - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { - // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report - // is none of this delivery's. - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) - says["anchor/app"] = applied - says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) - } - }) -} - -// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and -// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). -func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { - t.Helper() - says := map[string]inventory.ProvReport{} - wasSays := provAppliedOn - provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { - r, ok := says[node+"/"+module] - return r, ok, nil - } - inner := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - got, err := inner(ctx, open, names) - if err != nil { - return got, err - } - for _, n := range names { - if n != "anchor" { - continue - } - inv := open.inventory - s, _, err := inv.SentTo(ctx, n) - if err != nil { - return nil, err - } - builds, _, err := inv.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - plans := map[string]inventory.Provisioning{} - for m, d := range handed { - plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} - } - raw, err := json.Marshal(plans) - if err != nil { - return nil, err - } - id := nodeID(t, open, n) - if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { - return nil, err - } - } - return got, nil - } - t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) - return says -} diff --git a/cmd/mesh-controller/release_provisioned_test.go b/cmd/mesh-controller/release_provisioned_test.go deleted file mode 100644 index da197dfa..00000000 --- a/cmd/mesh-controller/release_provisioned_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a -// machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said -// so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done. - -// Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration -// hands to the provisioner. -func TestAWalkEndWaitsOnlyOnTheModulesItSent(t *testing.T) { - now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) - at := now.Add(-10 * time.Second) - handed := map[string]string{"app": "t1", "other": "t2"} - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ - "ace": {Digest: "d-ace", At: now.Add(-time.Minute), Provisioned: handed}}}}} - reports := []inventory.Reported{{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}} - says := map[string]inventory.ProvReport{"ace/app": {Status: inventory.ProvApplied, Templates: "t1", At: at}} - prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - // other has said nothing: not this walk's to wait for. - if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("missing: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - says["ace/other"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t2", Reason: "stuck", At: at} - if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("failed: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // once the walk carries it too, it does. - p.Modules["other"] = &inventory.PlanModule{Rest: p.Modules["app"].Rest} - if _, failed, _ := walkReports(p, reports, now, prov); len(failed) != 1 || !strings.Contains(failed[0], "other: stuck") { - t.Fatalf("a carried module's failure is not named: %v", failed) - } -} diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go deleted file mode 100644 index 6e052994..00000000 --- a/cmd/mesh-controller/release_test.go +++ /dev/null @@ -1,271 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one -// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236). - -type backlogMesh struct { - open *stores - sent [][]string - broken map[string]bool -} - -// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and -// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once. -func aBacklog(t *testing.T) *backlogMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - b := &backlogMesh{open: open, broken: map[string]bool{}} - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - - build := func(module, commit, made string, asked time.Time) { - manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}} - for _, m := range []string{"app", "late", "same"} { - build(m, "c1", "sha256:"+m+"-1", old) - for _, n := range on[m] { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - sent := map[string]string{} - for m, nodes := range on { - for _, x := range nodes { - if x == n { - sent[m] = "c1" - } - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { - t.Fatal(err) - } - } - build("app", "c2", "sha256:app-2", now) - build("late", "c2", "sha256:late-2", now) - build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes - - assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - machineMoves = assigned - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true}, nil - } - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - b.sent = append(b.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - // A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict). - outcome := inventory.OutcomeApplied - if b.broken[node] { - outcome = inventory.OutcomeFailed - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if b.broken[r.Node] { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - return f, nil - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - return b -} - -func (b *backlogMesh) release(t *testing.T) inventory.Plan { - t.Helper() - plans, err := b.open.inventory.RecentPlans(t.Context(), 10) - if err != nil { - t.Fatal(err) - } - for _, p := range plans { - if p.Release != nil { - return p - } - } - t.Fatal("no release plan") - return inventory.Plan{} -} - -// The backlog goes out one machine at a time: the first judged before the second is sent, each build's -// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. -func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - moves, _ := machineMoves(ctx, b.open, f, "laptop", false) - var names []string - for _, mv := range moves { - names = append(names, mv.Module) - } - if !reflect.DeepEqual(names, []string{"app", "late"}) { - t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names) - } - // Nothing else may carry them: a send that judges nothing is refused. - if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) { - t.Fatalf("a send that judges nothing would carry them: %v", err) - } - - advancePlans(ctx, b.open) - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent) - } - p := b.release(t) - if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil { - t.Fatalf("the release plan is %+v", p.Release) - } - gateEvery = 0 - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", b.sent) - } - p = b.release(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) { - t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, build := range []string{"build-app-c2", "build-late-c2"} { - if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err) - } - } - // Nothing waits now, and nothing opens again. - if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { - t.Fatalf("a release opened with nothing waiting: %+v %v", p, err) - } -} - -// A release that fails on its first machine puts back what it carried there, goes no further, and the -// next one waits for a person, said as a condition; a person releases it with why. -func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - - p := b.release(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the release is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("app is at %s, not put back", current["app"].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed at its gate") - } - // late still waits on laptop, and is not released on its own after a failure. - if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { - t.Fatalf("a release opened after a failed one: %+v %v", p, err) - } - if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") { - t.Fatalf("the held release is not said: %+v", obs) - } - b.broken["anchor"] = false - if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" { - t.Fatalf("the person's release is %s (%+v)", p.State, p.Release) - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("late was not released to laptop: %v", sent) - } -} diff --git a/cmd/mesh-controller/removed_module_test.go b/cmd/mesh-controller/removed_module_test.go deleted file mode 100644 index 298949d8..00000000 --- a/cmd/mesh-controller/removed_module_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// 1922/AC1: a tier naming a module the catalogue no longer holds asks it without failing the plan: the module ends -// as deleted (not built, not sent) and the tier's other modules go on. -func TestAModuleTheCatalogueNoLongerHoldsEndsDeletedAndTheTierGoesOn(t *testing.T) { - p, byName := aWalkOfOne("sudo") - p.Tiers = [][]string{{"gone", "sudo"}} - for _, m := range p.Tiers[0] { - askModule(t.Context(), p, m, byName) - } - if g := p.Modules["gone"]; g == nil || g.State != planDeleted || g.Build != "" || g.SentAt != nil { - t.Fatalf("the removed module is not deleted: %+v", g) - } - if p.State == inventory.PlanFailed { - t.Fatalf("the plan failed: %q", p.Note) - } - if s := p.Modules["sudo"]; s == nil || s.State != "asked" { - t.Fatalf("the other module of the tier did not go on: %+v", s) - } -} - -// 1922/AC2: a plan that failed on such a module before the fix is retried: the removed module ends deleted and the -// plan is building again, to go on to its next tier. -func TestARetryOfAPlanFailedOnARemovedModuleResumes(t *testing.T) { - p := inventory.Plan{ID: "plan-1", State: inventory.PlanFailed, Note: "gone is no longer in the catalogue", - Tiers: [][]string{{"gone"}, {"next"}}, - Modules: map[string]*inventory.PlanModule{ - "gone": {State: "failed", Why: "no longer in the catalogue"}, - }} - failed := toRetry(&p, nil, nil) - if len(failed) != 1 || failed[0] != "gone" { - t.Fatalf("the failed set is %v", failed) - } - for _, m := range failed { - askModule(t.Context(), &p, m, map[string]inventory.Entry{}) - } - resumed(&p, "retried") - if p.State != inventory.PlanBuilding || p.Modules["gone"].State != planDeleted { - t.Fatalf("the retry did not resume: %s %+v", p.State, p.Modules["gone"]) - } -} - -// 1922/AC3: an ended plan's note names the verb that releases the modules waiting for a gate, not `push --behind`. -func TestAnEndedPlansHintNamesTheReleaseOfTheBacklog(t *testing.T) { - p := inventory.Plan{State: inventory.PlanFailed, Note: "failed", - Modules: map[string]*inventory.PlanModule{"agent": {State: "built"}}} - sayUnsent(&p, func(string) bool { return true }) - if strings.Contains(p.Note, "push --behind") || !strings.Contains(p.Note, "upgrade release-backlog") { - t.Fatalf("the note reads %q", p.Note) - } -} diff --git a/cmd/mesh-controller/replay201_test.go b/cmd/mesh-controller/replay201_test.go deleted file mode 100644 index 6e37021b..00000000 --- a/cmd/mesh-controller/replay201_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestReplay201ThePlansSend replays novox/hq issue 201 (2026-10-02): two merges a minute apart made two plans; the -// controller's plan rolled the newer controller onto the control node, and the node-engine's plan then sent that -// machine the controller build before it, whose binary refused the seat row the newer one had widened. Here the -// control node, anchor, was last sent the newer controller c2 while the mesh's record of the controller is the -// build before it, c1, which passed its gate. The outcome asserted: a send that is no person's push — composed as a -// plan's send composes it (keepingRecorded, what sendKeeps gives every send through sendToEach) — carries the -// controller at c2, the build the machine was last sent, and not the older record; a person's push still carries -// the record. Written with only what the controller had before its fix (mesh-controller #273) — the inventory's -// build, gate, assignment and send records, keepingRecorded, planFor and declarationFor — so it is laid over the -// commit before; the fix's own TestReplay201 reads what the fix added. Needs the store (MESH_TEST_POSTGRES), and -// is named in testdata/store-tests. -func TestReplay201ThePlansSend(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - const older, newer = "c1aaaaaaaaaa", "c2bbbbbbbbbb" - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - record := func(commit string, asked time.Time) inventory.Build { - t.Helper() - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) - b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, - Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - return b - } - - // The older controller rolled out and passed its gate; the newer one was asked after it and registered. - before := record(older, old) - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: older, - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - record(newer, now) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-controller", BuiltFrom: newer, Head: newer, Asked: now}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - // The controller's plan sent anchor the newer controller. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": newer}); err != nil { - t.Fatal(err) - } - // And the mesh's record of the controller is the build before it — whatever made it so. - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - - // The controller each send to anchor carries: a plan's send keeps recorded builds (keepingRecorded, what - // sendKeeps gives every send but a person's push); a person's push composes the mesh's record. - controllerSent := func(sends string, keeping bool) string { - t.Helper() - c := ctx - if keeping { - c = keepingRecorded(ctx) - } - plan, settings, err := resolveNode(c, open, "anchor") - if err != nil { - t.Fatalf("%s to anchor did not compose: %v", sends, err) - } - declared, err := declarationFor(c, open, "anchor", plan, settings) - if err != nil { - t.Fatalf("%s to anchor did not compose: %v", sends, err) - } - return declared.Builds["mesh-controller"] - } - - if got := controllerSent("a plan's send", true); got != newer { - t.Fatalf("a plan's send carried anchor the controller %s, older than the %s anchor was last sent: the "+ - "older binary would read the row its successor widened", short(got), short(newer)) - } - if got := controllerSent("a person's push", false); got != older { - t.Fatalf("a person's push carried the controller %s, not the mesh's record %s", short(got), short(older)) - } -} diff --git a/cmd/mesh-controller/replay309_test.go b/cmd/mesh-controller/replay309_test.go deleted file mode 100644 index c7e46c5d..00000000 --- a/cmd/mesh-controller/replay309_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "reflect" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 309, replayed. The group `feat/a-machine-joins-through-the-tunnel` (2026-10-08): the controller's pull request -// moved the controller and the build agent, the node-engine's moved the node-engine, the lab's moved -// nothing. The build agent builds the node-engine (built by: the controller's member first), and the -// node-engine goes before the controller (engine before controller: the node-engine's member first). Both -// rules applied to one pair in opposite directions, and the group was rejected as a cycle; the person merged -// the controller first by hand. A rule read from the graph outranks the rollout default, so the group is -// ordered, the controller's member first. -func TestReplay309(t *testing.T) { - members := []orderMember{ - {ID: "novox/mesh-controller@8bbfdb53db2b", Repository: "novox/mesh-controller", Number: 132}, - {ID: "novox/mesh-host@86cbebd10f8a", Repository: "novox/mesh-host", Number: 52}, - {ID: "novox/mesh-lab@e80a4b1642bf", Repository: "novox/mesh-lab", Number: 61}, - } - reach := map[string]orderReach{ - "novox/mesh-controller@8bbfdb53db2b": {Moved: []string{"build-agent", "mesh-controller", "route-proxy"}, - Manifests: []string{"module.json"}}, - "novox/mesh-host@86cbebd10f8a": {Moved: []string{"mesh-host"}}, - "novox/mesh-lab@e80a4b1642bf": {}, - } - // Every source-built module is built by the build seat's holder; the holder follows the controller. - edges := []inventory.Edge{ - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - {From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - } - got := orderOf(members, reach, edges, map[string]moduleChange{"mesh-controller": {Manifest: true}}) - if len(got.Cycle) > 0 { - t.Fatalf("the group is refused as a cycle %v; pairs %+v", got.Cycle, got.Pairs) - } - // Being built by the build seat's holder orders nothing (ADR 0360 section 3, ADR 0419): the node-engine goes - // first by the rollout default, and no pair puts the controller's member before it. - want := []string{"novox/mesh-host@86cbebd10f8a", "novox/mesh-controller@8bbfdb53db2b", "novox/mesh-lab@e80a4b1642bf"} - if !reflect.DeepEqual(got.Order, want) { - t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs) - } -} diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go deleted file mode 100644 index 39ce20ec..00000000 --- a/cmd/mesh-controller/replay318_test.go +++ /dev/null @@ -1,257 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the -// older commit: the statements are read from the node-engine's JSON, as the controller hears them. -// -// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send — -// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged -// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the -// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service -// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its -// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and -// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there. -// -// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along, -// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence -// for the operator, that a new login is needed on the laptop. -func TestReplay318(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - keeper, _ := withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - - build := func(module, repository, commit string, asked time.Time) { - m := catalogue.Manifest{Module: module, Version: "1"} - if module == "openrazer" && commit == "c78b5fc9" { - // The build that put the operator's account in the group `openrazer` (ADR 0252). - m.Resources = []map[string]any{{"id": "operator", "type": "user", "name": "operator", - "groups": []any{"openrazer"}}} - } - manifest, _ := json.Marshal(m) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, - Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, m, inventory.Source{ - Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - type move struct{ module, repository, from, to string } - moves := []move{ - {"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"}, - {"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"}, - {"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"}, - } - machines := []string{"laptop", "workstation"} - if _, err := inv.AddNode(ctx, "workstation"); err != nil { - t.Fatal(err) - } - old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute) - for _, mv := range moves { - build(mv.module, mv.repository, mv.from, old) - for _, n := range machines { - if _, err := inv.Assign(ctx, n, mv.module); err != nil { - t.Fatal(err) - } - } - } - for _, n := range machines { - sent := map[string]string{} - for _, mv := range moves { - sent[mv.module] = mv.from - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { - t.Fatal(err) - } - } - for _, mv := range moves { - build(mv.module, mv.repository, mv.to, recent) - } - - // What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the - // daemon's unit failed in that account's own manager; docker's container is healthy. - statement := func(at time.Time) link.Health { - raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [ - {"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service", - "state": "healthy", "since": %q}, - {"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator", - "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, - "reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"}, - {"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service", - "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, - "reason": "failed in the account's own service manager (exit-code)"}]}`, - link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), - at.Format(time.RFC3339Nano)) - var h link.Health - if err := json.Unmarshal([]byte(raw), &h); err != nil { - t.Fatal(err) - } - return h - } - - wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"laptop": true, "workstation": true}, nil - } - var sends [][]string - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - // Every judging hears each machine's newest statement first, as the controller does between judgings: - // the account's wait and the failed unit are raised as openrazer's condition, after the send. - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - now := time.Now() - f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{}, - judged: true} - for _, m := range machines { - if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil { - return f, err - } - f.served[m] = served{runtime: true, tools: map[string]bool{}} - } - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - } - if f.health, err = open.inventory.Healths(ctx); err != nil { - return f, err - } - f.open, f.openErr = keeper.Open(ctx) - return f, nil - } - // The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once. - gateWithin, gateEvery, gateBound = time.Hour, 0, 0 - - for i := 0; i < 12; i++ { - advancePlans(ctx, open) - time.Sleep(5 * time.Millisecond) - } - - var p inventory.Plan - plans, err := inv.RecentPlans(ctx, 10) - if err != nil { - t.Fatal(err) - } - for _, q := range plans { - if q.Release != nil { - p = q - break - } - } - if p.Release == nil { - t.Fatal("no walk of the builds waiting for a gate was opened") - } - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) { - t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State, - p.Release.Done, p.Note) - } - if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) { - t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - if current["openrazer"].Commit != "c78b5fc9" { - t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit) - } - for _, mv := range moves { - v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to) - if err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err) - } - if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") { - t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why) - } - } - // What the operator reads: one condition per machine for openrazer, saying a new login is needed there, - // a warning for a person, and no fault of the build. - list, err := keeper.Open(ctx) - if err != nil { - t.Fatal(err) - } - var said []string - for _, c := range list { - said = append(said, c.Key+": "+c.Summary) - if c.Subject.ID == "openrazer.laptop" { - if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent || - c.Resolver != conditions.ResolverOperator { - t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary) - } - } - } - if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") { - t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n")) - } -} diff --git a/cmd/mesh-controller/replay325_test.go b/cmd/mesh-controller/replay325_test.go deleted file mode 100644 index b4dacb4f..00000000 --- a/cmd/mesh-controller/replay325_test.go +++ /dev/null @@ -1,57 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "slices" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq issue 325, replayed with only what the controller had before its fix, so it can be laid over the -// older commit. On 2026-10-08 the catalogue's pull request adding `sensors` merged, and the merge asked for -// its build (issue 300). About a minute later `assign g14 sensors` answered "no module of that name: -// sensors"; a few minutes later the same call worked, because the build had registered the module. The -// answer read as "nobody registered it". An assignment made while the build runs says the build — where it -// is from, since when, its id — is kept, and is made when the build registers the module. -func TestReplay325(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - // The build the merge asked before R4e (nothing asks one since, ADR 0501 §6), kept as it was. - mergeAdding(t, open, "modules/sensors", "5e450125aa") - - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil { - t.Fatalf("assign while the merge's build runs was refused: %v", err) - } - for _, want := range []string{"being built", "novox/mesh-catalog", "modules/sensors", "b-modules/sensors"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - - manifest, _ := json.Marshal(catalogue.Manifest{Module: "sensors", Version: "1"}) - if err := (builds{open.inventory, open}).Built(ctx, link.BuildResult{ID: "b-modules/sensors", - Repository: "novox/mesh-catalog", Path: "modules/sensors", Ref: "main", Commit: "5e450125aa", On: "anchor", - Module: "sensors", Manifest: manifest, - Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}}); err != nil { - t.Fatal(err) - } - assigned, err := open.inventory.Assigned(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if !slices.Contains(assigned, "sensors") { - t.Fatalf("the build registered sensors and laptop runs %v", assigned) - } -} diff --git a/cmd/mesh-controller/replay335_integration_test.go b/cmd/mesh-controller/replay335_integration_test.go deleted file mode 100644 index 98d142a2..00000000 --- a/cmd/mesh-controller/replay335_integration_test.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build integration - -package main - -import ( - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "m01", "m02") - ctx := t.Context() - inv := tm.open.inventory - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the tier", tm.sent) - } - p := tm.plan(t) - lead, carried := p.Modules["m01"], p.Modules["m02"] - if lead.Gate == nil || carried.GatedBy != "m01" { - t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried) - } - // The lead passed; the carried module's turn did not come. The first send is past the wait's bound. - sent := time.Now().UTC().Add(-planWaitBound - time.Minute) - judged := sent.Add(2 * time.Minute) - lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent - lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed, - "healthy 3 times over 2m5s", &judged, true - carried.Gate = nil - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - // Another walk's send reached anchor, which has not reported on it. - if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere", - map[string]string{"m01": "c2", "m02": "c2"}); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, tm.open) - p = tm.plan(t) - if p.State == inventory.PlanFailed { - t.Fatalf("the plan failed after its gate passed: %s", p.Note) - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: the rest once, as the pass said", tm.sent) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - for _, m := range []string{"m01", "m02"} { - if current[m].Commit != "c2" { - t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Errorf("%s's build was marked failed at its gate after the gate passed", m) - } - } - if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Errorf("m02 did not take over m01's pass: %+v", g) - } -} diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go deleted file mode 100644 index 5cd51649..00000000 --- a/cmd/mesh-controller/replay335_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestReplay335 replays novox/hq issue 335 (2026-10-08): dunst's new build passed its gate on the laptop -// (healthy 3 times over 2m5s); its send to the rest was refused while another walk's build waited on the -// workstation; that walk then sent the laptop a declaration the laptop did not report on; and thirty minutes -// after the first send the plan read the laptop's silence as the passed build never applied, marked it failed -// at its gate with the pass's own words, and put it back. Written with only what the controller had before -// its fix, so it is laid over the commit before. -func TestReplay335(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports) - t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass) -} - -// novox/hq issue 335: a build that passed its gate on its first machine is not judged again from that -// machine's later reports. On 2026-10-08 the send to the rest waited on another walk, that walk sent the -// first machine a declaration it did not report for half an hour, and the plan failed the passed build at -// the wait's bound and put it back. -func testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports(t *testing.T) { - sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC) - judged := sentAt.Add(2 * time.Minute) - later := sentAt.Add(5 * time.Minute) - state := inventory.PlanModule{First: []string{"laptop"}, FirstAt: &sentAt, - Gate: &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed, - Why: "healthy 3 times over 2m5s", JudgedAt: &judged, Kept: true}} - running := []string{"laptop", "workstation"} - now := sentAt.Add(30*time.Minute + 9*time.Second) - - for what, reports := range map[string][]inventory.Reported{ - "no report about what it was sent since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeApplied, Current: false}}, - "another send failed there since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeFailed, Current: true}}, - "no report at all": nil, - } { - step := nextRollout(state, running, false, reports, now, 30*time.Minute) - if step.failed != "" || step.waiting != "" || !reflect.DeepEqual(step.send, []string{"workstation"}) { - t.Errorf("%s: %+v, want the rest sent as the pass said", what, step) - } - } -} - -// novox/hq issue 335: whatever stops a walk after its gate passed, the passed build is not marked failed at -// its gate, nor put back. -func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { - sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed, Why: "healthy 3 times over 2m5s"} - state := &inventory.PlanModule{Build: "build-1", First: []string{"laptop"}, FirstAt: &sentAt, Gate: g} - p := &inventory.Plan{ID: "plan-1", Modules: map[string]*inventory.PlanModule{"dunst": state}} - // No stores: a walk that keeps the pass touches none, and one that reaches for them is putting it back. - defer func() { - if r := recover(); r != nil { - t.Fatalf("the passed build was taken to be failed and put back: %v", r) - } - }() - failFirstSend(t.Context(), nil, p, "dunst", state, []string{"laptop"}, "laptop did not report it applied within 30m0s", - []string{"workstation"}) - if g.Verdict != inventory.GatePassed || g.Rollback != "" { - t.Fatalf("the passed gate became %q, rollback %q", g.Verdict, g.Rollback) - } - if strings.Contains(p.Note, "failed its gate") || strings.Contains(p.Note, "put back") || - !strings.Contains(p.Note, "did not report it applied") { - t.Fatalf("the note reads %q", p.Note) - } -} - -// novox/hq issue 335 review: **a module carried by its lead's send takes over the lead's pass before its own -// step is read.** The state is the one a step leaves when the lead's gate passed and the step ended before the -// carried module's turn (an error read before it, kept with the plan): the lead passed, the carried module has -// no gate of its own yet. Since then another send reached the first machine and it has not reported on it, and -// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the -// carried module's step said the first machine never applied it, and the passed build was put back. diff --git a/cmd/mesh-controller/replay336_test.go b/cmd/mesh-controller/replay336_test.go deleted file mode 100644 index 29471508..00000000 --- a/cmd/mesh-controller/replay336_test.go +++ /dev/null @@ -1,95 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestReplay336 replays novox/hq issue 336 (2026-10-08): a catalogue merge built a new bus build for the -// control-node; from then on every send to that machine was refused for the bus's planned step, which only a -// person starts, and for 28 minutes nothing but the walks' notes said so. The outcome asserted: the first -// watchdog tick after the first refusal opens a condition for the operator that names what waits and the verb -// that ends it, and it clears once the bus's machine runs the new build. Written with only what the controller -// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and -// seeing — so it is laid over the commit before. -func TestReplay336(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}} - if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/nats", BuiltFrom: "32307bd1", Head: "32307bd1"}); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "engine", Version: "1"}) - for _, m := range []string{"nats", "engine"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - // The control-node runs the bus build it was last sent; the mesh holds a newer one, which waits for its step. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "88135ad0"}); err != nil { - t.Fatal(err) - } - // A walk of the engine, built, whose tier sends to the control-node. - now := time.Now().UTC() - plan := inventory.Plan{ID: "plan-engine", Repository: "novox/mesh-host", Commit: "e1e1e1e1", Created: now, - State: inventory.PlanBuilding, Tiers: [][]string{{"engine"}}, - Modules: map[string]*inventory.PlanModule{"engine": {State: "built", BuiltAt: &now, Commit: "e1e1e1e1", Build: "b"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - p, err := inv.PlanByID(ctx, "plan-engine") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(p.Note, errBusWaits.Error()) { - t.Fatalf("the walk's send to the bus's machine was not refused for the bus: %s %q", p.State, p.Note) - } - - store := conditions.NewInMemory() - k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: &conditions.Told{}}) - defer k.Close(context.Background()) - w := &watchdogs{open: open, keeper: k, started: now.Add(-time.Hour)} - waiting := func() []conditions.Condition { - t.Helper() - w.see(ctx, w.gather(ctx)) - all, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var out []conditions.Condition - for _, c := range all { - if strings.HasPrefix(c.Key, "bus.") && c.Resolver == conditions.ResolverOperator { - out = append(out, c) - } - } - return out - } - got := waiting() - if len(got) != 1 { - t.Fatalf("the first tick after the refusal told the operator nothing about the bus's step: %d condition(s)", len(got)) - } - for _, want := range []string{"anchor", "engine", "mesh-controller.bus", "upgrade", "32307bd1"} { - if !strings.Contains(got[0].Summary, want) { - t.Errorf("the condition does not say %q: %s", want, got[0].Summary) - } - } - // The step taken: the control-node is sent the new bus build, and the wait is over. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", map[string]string{"nats": "32307bd1"}); err != nil { - t.Fatal(err) - } - if got := waiting(); len(got) != 0 { - t.Fatalf("the bus's machine runs the new build, and the operator is still asked: %+v", got) - } -} diff --git a/cmd/mesh-controller/replay348_test.go b/cmd/mesh-controller/replay348_test.go deleted file mode 100644 index 30f23918..00000000 --- a/cmd/mesh-controller/replay348_test.go +++ /dev/null @@ -1,96 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// Replay of novox/hq issue 348 (2026-10-09): the control node's resolver stopped answering on its private -// address at 10:57:57 UTC, and the machine's network condition was raised at 10:58:45. The controller and the -// node-engine were sent to their first machines at 10:59:34. The restarted node-engine's first statement judged -// the names part once — unknown, "one look failed; a second decides" — and that statement cleared the condition, -// though the resolver still refused. Its next look raised it again at 11:00:23, after the send, and at 11:10:29 -// both builds failed their gate with "raised since it was sent: machine..network" and were put -// back, for a fault that began before they were sent. -// -// Written only with what the controller had before its fix (mesh-controller #179): a mesh, the machine's -// statements as stateHealth keeps them, the keeper's open conditions and the gate's aboutTheMachine — so the -// prover (mesh-lab replays/cmd/prove) lays it over the commit before. Its outcome is the operator's acceptance: -// a build is never put back for a problem that started before it was sent; and the fault is not forgotten -// while the resolver still refuses. Needs a store: MESH_TEST_POSTGRES. -func TestReplay348AFaultFromBeforeTheSendFailsNoGate(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - const key = "machine.anchor.network" - names := func(state string, streak int) link.NetworkPart { - p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} - if state == link.StateUnhealthy { - p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" - p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" - p.Toward = []string{"10.77.0.1"} - } - return p - } - say := func(at time.Time, state string, streak int) { - t.Helper() - n := &link.NetworkHealth{State: state, Since: h0, Parts: []link.NetworkPart{names(state, streak)}} - if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - network := func() (conditions.Condition, bool) { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key == key { - return c, true - } - } - return conditions.Condition{}, false - } - - // 10:58:45 — the node-engine's second failing look: raised. - say(h0, link.StateUnhealthy, 2) - if _, ok := network(); !ok { - t.Fatal("the resolver refusing every question on the control node raised nothing") - } - // 10:59:34 — the controller and the node-engine are sent. - time.Sleep(10 * time.Millisecond) - sent := time.Now().UTC() - time.Sleep(10 * time.Millisecond) - - // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. - say(h0.Add(time.Minute), link.StateUnknown, 1) - if _, ok := network(); !ok { - t.Error("the restarted node-engine's first, undecided statement cleared the network condition while " + - "the resolver still refused") - } - // 11:00:23 — its second look: unhealthy again. - time.Sleep(10 * time.Millisecond) - say(h0.Add(2*time.Minute), link.StateUnhealthy, 2) - if _, ok := network(); !ok { - t.Fatal("the resolver still refusing raised nothing") - } - - // 11:10:29 — the gate on the control node, for the builds sent after the fault began. - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, moved := range [][]string{{"mesh-host"}, {"mesh-controller"}} { - w := aboutTheMachine("anchor", moved, sent, gateFacts{judged: true, open: list}) - if w.whole != "" || len(w.on) != 0 { - t.Errorf("%s, sent after the resolver stopped answering, was held for that fault: whole %q, on %v", - moved[0], w.whole, w.on) - } - } -} diff --git a/cmd/mesh-controller/replay349_test.go b/cmd/mesh-controller/replay349_test.go deleted file mode 100644 index 930961fb..00000000 --- a/cmd/mesh-controller/replay349_test.go +++ /dev/null @@ -1,108 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// Replay of novox/hq issue 349 (2026-10-09): two merges into the catalogue's main were acted on in reverse order. -// The later one (a082615b, a security fix to the forge's module) was planned first; the earlier one (8ff8197a, -// another module), handed over late, was planned after it, and its plan superseded the later merge's at tier 0 -// with "gitea planned there again": the forge's module was to be built for the commit without the fix. Written -// only with what the controller had before its fix (mesh-controller #179): a mesh, two registered modules of the -// catalogue, the merges as the forge announces them, the plans the inventory keeps — so it is laid over the -// commit before. Since novox/hq ADR 0276 the earlier merge waits in a batch behind the later merge's walk and is -// planned only when that walk ends and the cutter looks again; the replay ends the walk and has a merge of another -// repository heard, which looks, and a plan of the two merges left standing with nothing planned fails it. -func TestReplay349(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - for _, module := range []string{"app", "notes"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "other", Version: "1"}, - inventory.Source{Repository: "novox/other", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - // Both made after the modules were last looked at, so neither is history. - made := time.Now().UTC().Add(time.Minute) - merge := func(commit, module string, at time.Time) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, - MergedAt: at.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"}, - ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true} - } - const fix, earlier = "a082615bfix", "8ff8197aearlier" - // The later merge, the fix to notes, is acted on first; the earlier one, moving app, after it. - if err := (following{open: open}).SourceMoved(ctx, merge(fix, "notes", made.Add(time.Minute))); err != nil { - t.Fatal(err) - } - if err := (following{open: open}).SourceMoved(ctx, merge(earlier, "app", made)); err != nil { - t.Fatal(err) - } - - // The later merge's plan builds and ends; then a merge of another repository comes. Whatever holds the earlier - // merge back while that plan runs (a batch queued behind it, novox/hq ADR 0276) is decided by now: every plan - // of the two merges left standing says what it builds. - plans, err := inv.RecentPlans(ctx, 20) - if err != nil { - t.Fatal(err) - } - for _, p := range plans { - if p.Commit == fix && p.Open() { - p.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - } - } - if err := (following{open: open}).SourceMoved(ctx, link.SourceMoved{Owner: "novox", Repo: "other", Base: "main", - Commit: "c1other", MergedAt: made.Add(2 * time.Minute).Format(time.RFC3339Nano), - Paths: []string{"main.go"}}); err != nil { - t.Fatal(err) - } - if plans, err = inv.RecentPlans(ctx, 20); err != nil { - t.Fatal(err) - } - // A plan superseded hands its modules to the plan that took it over; the plans left standing are what the mesh - // builds. Each plan of the two merges left standing plans modules; one builds app, the earlier merge's module; - // one builds notes, the module the later merge fixed, and every one that does builds it at the commit holding - // the fix. - app, notes := 0, 0 - for _, p := range plans { - t.Logf("%s %s at %.8s: %v %s", p.ID, p.State, p.Commit, p.Tiers, p.Note) - if p.State == inventory.PlanSuperseded || (p.Commit != fix && p.Commit != earlier) { - continue - } - if len(p.Modules) == 0 { - t.Errorf("%s, of the merge at %s, is left standing and plans nothing: %s", p.ID, p.Commit, p.Note) - } - if _, holds := p.Modules["app"]; holds { - app++ - } - if _, holds := p.Modules["notes"]; holds { - notes++ - if p.Commit != fix { - t.Errorf("%s plans the fixed module notes at %s, a commit without the fix %s: %s", p.ID, p.Commit, fix, - p.Note) - } - } - } - if app == 0 { - t.Error("no plan left standing builds app, the module the earlier merge moved") - } - if notes == 0 { - t.Error("no plan left standing builds notes, the module the later merge fixed") - } -} diff --git a/cmd/mesh-controller/replay614_test.go b/cmd/mesh-controller/replay614_test.go deleted file mode 100644 index 4c2f9d9a..00000000 --- a/cmd/mesh-controller/replay614_test.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build integration - -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A new SDK plans the toolchain, and then every bundle compiled in it** (novox/hq issue 614), reasoned -// as `mesh-controller.delivery-plan` reasons it: the modules registered as a build registers them — -// resolved, so carrying no `build` — their builds recorded with what they stood on and what they made, -// the graph read by theGraph and the change scoped by pullScope. On 2026-10-13 mesh-sdk 0.1.14 planned -// "builds mesh-sdk, sent nowhere": mesh-tools' `build.on` naming the SDK's package and the providers' -// TypeScript bundles were edges only while their manifests were unbuilt. -// -// **The replay of issue 614** (mesh-lab R614): written only with what the controller had before its fix — a -// store, modules registered and builds recorded, theGraph and pullScope — so it is laid over the commit before -// and fails there with the plan of that day, [[mesh-sdk]]. Needs a store: MESH_TEST_POSTGRES. -func TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles(t *testing.T) { - inv := inventory.ForTest(t) - ctx := t.Context() - asked := time.Now().Add(-time.Hour) - digest := func(c string) string { return "@sha256:" + strings.Repeat(c, 64) } - register := func(m catalogue.Manifest, repository, path string, against []string, made []inventory.Artifact) { - t.Helper() - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path, - Ref: "main", BuiltFrom: "old", Asked: asked}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main", - Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Made: made, Asked: asked}); err != nil { - t.Fatal(err) - } - } - bundle := func(module, language string) catalogue.Manifest { - return catalogue.Manifest{Module: module, Version: "1", Bundles: []catalogue.Bundle{{ - Name: "code", Language: language, Source: catalogue.ArtifactStoreScheme + module + "/code" + digest("2"), - Entrypoints: []string{"tools/index.js"}}}} - } - register(catalogue.Manifest{Module: "build-agent", Version: "1", - Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, - "novox/mesh-catalog", "modules/build-agent", nil, nil) - register(catalogue.Manifest{Module: "mesh-sdk", Version: "1"}, "novox/mesh-sdk", "", nil, - []inventory.Artifact{{Name: "lib", Kind: "package", Reference: "@novox/mesh-sdk@0.1.13"}}) - register(catalogue.Manifest{Module: "mesh-tools", Version: "1"}, "novox/mesh-tools", "", - []string{"@novox/mesh-sdk@0.1.13", catalogue.ArtifactStoreScheme + "upstream/docker.io/library/node" + digest("0")}, - []inventory.Artifact{{Name: "build", Kind: "image", Reference: catalogue.ArtifactStoreScheme + "mesh-tools/build" + digest("1")}}) - register(catalogue.Manifest{Module: "mesh-tools-go", Version: "1"}, "novox/mesh-tools", "go", nil, nil) - register(bundle("minio", "typescript"), "novox/mesh-catalog", "modules/minio", nil, nil) - register(bundle("umami", "typescript"), "novox/mesh-catalog", "modules/umami", nil, nil) - register(bundle("dnsmasq", "go"), "novox/mesh-catalog", "modules/dnsmasq", nil, nil) - - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - t.Fatal(err) - } - m := orderMember{ID: "novox/mesh-sdk@new", Repository: "novox/mesh-sdk", Head: "new", - Paths: []string{"src/provisioner/index.ts", "package.json"}} - s := pullScope(m.pull(), entries, read, edges) - // Since R4g no bundle is compiled in a toolchain image (526eed99, issue 1851 C1b: the toolchain edge is gone), so a - // new SDK plans itself and what stands on its package; the TypeScript bundles move by their own build keys, whose - // fingerprints name what they are built from (issue 1938: this test was kept from older runs until then). - want := [][]string{{"mesh-sdk"}, {"mesh-tools"}} - if !reflect.DeepEqual(s.Reach.Plan.Tiers, want) { - t.Fatalf("a new SDK plans %v, want %v (edges %v)", s.Reach.Plan.Tiers, want, edges) - } - if !reflect.DeepEqual(s.Dependents, []string{"mesh-tools"}) { - t.Errorf("dependents %v", s.Dependents) - } -} diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go deleted file mode 100644 index 76f28db8..00000000 --- a/cmd/mesh-controller/replays_integration_test.go +++ /dev/null @@ -1,473 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", - Requires: []string{"wildcard-resolution"}}) - for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - // The consumer's machine composes first, as a push does: what it is sent is what its provider grants. - for _, node := range []string{"laptop", "anchor"} { - plan, settings, err := resolveNode(ctx, open, node) - if err != nil { - t.Fatalf("%s does not resolve: %v", node, err) - } - if _, err := declarationFor(ctx, open, node, plan, settings); err != nil { - t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+ - "mesh_laptop_networkmanager (26 characters), refused it: %v", node, err) - } - } -} - -func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - for _, m := range []catalogue.Manifest{ - {Module: "store", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}, - Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}}, - {Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}, - {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}}, - {Module: "board", Version: "1", Requires: []string{"postgres-database"}}, - } { - register(t, open, m) - } - assignAll := func(pairs ...[2]string) { - for _, a := range pairs { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - } - // The control node holds the mesh's store and resolver seats; the home server runs its own of each. - assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) - for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { - if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { - t.Fatal(err) - } - } - assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, - [2]string{"laptop", "board"}) - - plan, _, err := resolveNode(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var board, network string - for _, n := range plan.Needs { - switch { - case n.For == "board" && n.Name == "postgres-database": - board = n.From - case n.For == "network" && n.Name == "wildcard-resolution": - network = n.From - } - } - if board != "laptop" { - t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+ - "database there (issue 273)", board) - } - if network != "anchor" { - t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network) - } -} - -func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` + - `"streak":5,"restarts":2}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the crash loop: %v", err) - } - crashLoop = raw - } - // `null` is an engine that states nothing — older than the judging — and its reports carry no health. - var stated map[string]any - if err := json.Unmarshal(crashLoop, &stated); err != nil { - t.Fatal(err) - } - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - registerAt := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - registerAt("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - registerAt("c2", time.Now().Add(-time.Minute)) - - // The machine: what it is sent is applied, and its report says what runs is starting. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - sequence := int64(0) - say := func(node, digest, state string) { - t.Helper() - sequence++ - health := map[string]any{} - for k, v := range stated { - health[k] = v - } - health["at"] = time.Now().UTC().Format(time.RFC3339Nano) - if state != "" && stated != nil { - resources := []any{} - for _, r := range stated["resources"].([]any) { - kept := map[string]any{} - for k, v := range r.(map[string]any) { - kept[k] = v - } - kept["state"], kept["reason"] = state, "" - resources = append(resources, kept) - } - health["resources"] = resources - } - said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest, - "report_sequence": sequence} - if stated != nil { - said["health"] = health - } - body, _ := json.Marshal(said) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - var sent [][]string - last := map[string]string{} - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - last[node] = digest - say(node, digest, "starting") - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, 0 - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, open) // the first machine is sent the new build, and starts it - if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) { - t.Fatalf("sent %v, not the first machine first", sent) - } - advancePlans(ctx, open) // judged while it starts - // Its container exits at start, and the runtime restarts it: the machine says so, again and again. - for i := 0; i < 6 && len(sent) == 1; i++ { - say("anchor", last["anchor"], "") - advancePlans(ctx, open) - } - gateBound = -time.Second // and the bound passes - advancePlans(ctx, open) - - p, err := inv.PlanByID(ctx, "plan-crashloop") - if err != nil { - t.Fatal(err) - } - gate := p.Modules["app"].Gate - for _, names := range sent { - if slices.Contains(names, "laptop") { - t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate) - } - } - if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed { - t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v", - p.State, p.Note, gate) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } -} - -func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` + - `"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the silent web application: %v", err) - } - silent = raw - } - var h link.Health - if err := json.Unmarshal(silent, &h); err != nil { - t.Fatal(err) - } - if h.Contract == 0 { - t.Fatal("the engine states nothing of the silent web application: it is older than the judging") - } - open1 := func() []conditions.Condition { - t.Helper() - list, err := conditionsFrom.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - for look := 1; look <= 2; look++ { - at := time.Now().Add(time.Duration(look) * time.Second) - h.At = at - if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil { - t.Fatal(err) - } - raised := open1() - switch { - case look == 1 && len(raised) != 0: - t.Fatalf("one look raised %v", raised[0].Key) - case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"): - t.Fatalf("two looks in a row did not raise the module's condition: %v", raised) - case look == 2: - t.Logf("raised on the second look: %s", raised[0].Summary) - } - } -} - -func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - // app's key is not built, so the plan stands on its tier as it did while the build ran (see plan_clock_integration_test.go). - holdKeys(t, "app") - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 0 { - t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" || - !before.Modules["app"].Key || before.Modules["app"].KeyAt != "3ad22356aa" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } -} - -func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299", - Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}} - if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil { - t.Fatal(err) - } - rows, err := inv.Seats(ctx) - if err != nil { - t.Fatal(err) - } - // The working set as the serving controller holds it: the store's rows, the compiled seats' marks. - // A seat the compiled set does not know keeps its mark only as defined here. - for i := range rows { - if rows[i].Name == catchingUp.Name { - rows[i].Serves = catchingUp.Serves - } - } - catalogue.UseSeats(rows) - t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) - - seats := []string{catchingUp.Name} - if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 { - allOptional := true - for _, v := range uplink.Serves { - allOptional = allOptional && v.Optional - } - if allOptional { - seats = append(seats, "node-uplink") - } else { - t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is") - } - } - - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"}) - now := time.Now() - var heard []machineFacts - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil { - t.Fatal(err) - } - for _, seat := range seats { - if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil { - t.Fatal(err) - } - } - heard = append(heard, machineFacts{name: node, lastHeard: now}) - } - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}} - - found, err := probeHolders(ctx, d) - if err != nil { - t.Fatal(err) - } - for _, o := range found { - for _, seat := range seats { - if strings.HasPrefix(o.ID, seat+".") { - t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said) - } - } - } -} - -func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64) - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {image, ""}}))); err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil { - t.Fatal(err) - } - } - // The merge: the resolver's new build, registered and held for a person's word. - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord, - start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil { - t.Fatal(err) - } - - conn := onATestBus(t) - // The operator's yes, machine by machine, as the seat's push records it before it sends. - for _, node := range []string{"anchor", "laptop"} { - set := flag.NewFlagSet("push", flag.ContinueOnError) - why := addHandActFlags(set) - if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node, - "--cause", "split-dns"}); err != nil { - t.Fatal(err) - } - why.record(ctx, "push", []string{node}) - } - acts, err := link.HandActs(ctx, conn, start) - if err != nil { - t.Fatal(err) - } - if len(acts) != 2 { - t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts) - } - f := calm(time.Now()) - f.handActs = acts - for _, o := range watchHandActs(f) { - if strings.Contains(o.ID, "split-dns") { - t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary) - } - } -} diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go deleted file mode 100644 index b9c93fe2..00000000 --- a/cmd/mesh-controller/replays_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what -// happened, asserting the rule's outcome rather than the fix's mechanism, so it can be run against the -// commit before the fix and fail there, and against the fix and pass. **Written only with what the -// controller had before each fix** — the stores, register, assign, planFor, declarationFor — so the -// prover (mesh-lab replays/cmd/prove) can lay this file over the older commit and run it there. -// -// Registered in mesh-lab's replays/register.go with the fix each one proves; run by this repository's -// merge check on every pull request with the rest of the suite. - -// **R263 — a consumer's identity never refuses its provider's machine.** On 2026-10-06 the network -// manager came to require the mesh's resolver; on a machine whose name made its identity 23 to 26 -// characters against the one global bound of 20, the anchor — the resolver's holder, carrying every -// consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the -// provider's machine composes whatever its consumers are called, and a provision that mints no -// credential holds no consumer to a key's length. - -// **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for -// the resolver re-bound every database consumer on the home server — which runs its own store, beside -// its applications' data — to the store seat's holder on the control node, which made each a new empty -// database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a -// machine running its own store stays bound to it while another machine holds the store's seat; the -// resolver, which every holder answers alike, follows its seat. - -// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq -// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the -// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the -// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container -// exits at start never passes its gate on the first machine, is put back there at the bound, and never -// reaches the second. -// -// The machine is heard as a node-engine says it: its report of the apply, then the same account said again -// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the -// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the -// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of -// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is -// written only with what the controller had before the judging, for the prover to lay over that commit. - -// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq -// ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its -// program ran while every request hung, and the mesh said its machine was healthy; a person found it. -// Liveness cannot see it and a TCP check cannot either: the port is open. The module's declared HTTP check -// can. The engine's half (mesh-host internal/liveness TestReplaySilentWebAppIsSaidUnhealthy) states what it -// found looking at such a program; here the controller hears that statement on two looks in a row and -// raises the module's condition — the second, never the first. `null` is an engine older than the -// readiness check: it states the program alive, and nothing is raised. - -// **R296 — a plan's clock counts from its tier, not from its last save.** On 2026-10-07 a plan for a -// merge to the catalogue, its one module's build asked at 19:48:41 and queued at the build seat for -// minutes, read "tier 1 of 1, building for 1s", then 4s, then 9s: the line counted from the plan's last -// save, and every advance — on the 30-second tick and after every build outcome on the mesh — saved it -// whether or not anything moved. A clock reset by each save is never LATE, so `plans` and `status` could -// not say a plan was stuck. The outcome asserted: a plan in its tier for forty minutes, saved seconds ago, -// says forty minutes and LATE. -func TestReplay296APlansLineCountsFromWhenItEnteredItsTier(t *testing.T) { - now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC) - entered := time.Date(2026, 10, 7, 19, 48, 41, 0, time.UTC) - asked := entered - p := inventory.Plan{ID: "plan-296", Repository: "novox/mesh-catalog", Branch: "main", Commit: "3ad22356", - State: inventory.PlanBuilding, Created: entered, TierEntered: entered, Updated: now.Add(-4 * time.Second), - Tiers: [][]string{{"app"}}, Modules: map[string]*inventory.PlanModule{"app": {State: "asked", AskedAt: &asked}}} - line := planLine(p, now) - if !strings.Contains(line, "for 41m19s") || !strings.Contains(line, "LATE") { - t.Fatalf("a plan in its tier since 19:48:41, read at 20:30:00 and saved 4s before, reads %q", line) - } -} - -// **R296, the save.** The same live case from the store's side: a plan whose only build was asked and -// still waits is advanced as the tick and the mesh's build outcomes advance it, and is not written again — -// its revision and its last save stay where its last change left them. Before the fix each advance saved -// it, and its revision went from 3 to 6. - -// **R299 — a holder of a seat whose verbs are all optional is not silent.** On 2026-10-07 the -// node-uplink seat gained its first verbs, `resolvers` and `links`, both optional while its holders catch -// up (ADR 0246, step 1); its holders served neither, so they registered nothing on the bus's discovery for -// it, and the self-check's D3 raised `seat.node-uplink..silent` on every machine of the mesh, -// which the healer then acted on. The outcome asserted: through D3 itself — the seats read back from the -// store as the controller reads them, a holder on record on every machine heard from, and nothing -// answering the discovery — no holder of a seat whose verbs are all optional is said silent. -// -// The live seat is replayed while its verbs are all optional; beside it, a seat whose verbs are all -// optional by definition, so the outcome is still held once node-uplink's verbs are required (step 3). - -// **R300 — a merge that adds a module builds it.** On 2026-10-07 a pull request to the catalogue added -// one module, systemd-resolved; its delivery plan said "builds new: modules/systemd-resolved, sent -// nowhere", and at the merge the controller said it "changed nothing any module the mesh holds is built -// from" and built nothing, so the module was never registered and `assign` refused it until a person -// built it by hand. The outcome asserted: the merge asks the build seat for the new module's directory, at -// the branch merged into, from the repository as the mesh spells it, and opens no plan for it. - -// **R301 — a person's push of a held recorded build is no repair.** On 2026-10-07 the operator approved, -// machine by machine, new builds of modules whose upgrade policy records rather than rolls out — the -// resolver, the FortiClient adapter, the network managers, the packet filter, mail. A recorded build -// moves only by a person's push (ADR 0242), so each was `push ` through the seat with the -// operator's yes as its why and a word for the change as its cause. The hand-act log counted every one -// as a repair, and S15 raised `mesh.hand-acts.split-dns.healer-wanted` ("repaired by hand 4 times in 14 -// days … a healer is wanted"), and the same for `words` and `uplink-verbs`. The outcome asserted: a push -// that carries only held recorded builds, recorded as the push records it, wants no healer however -// often the person gives that word — through S15 itself, reading the log as the watchdog reads it. diff --git a/cmd/mesh-controller/rest_at_once_integration_test.go b/cmd/mesh-controller/rest_at_once_integration_test.go deleted file mode 100644 index 0c10c878..00000000 --- a/cmd/mesh-controller/rest_at_once_integration_test.go +++ /dev/null @@ -1,293 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -func aBacklogOnThree(t *testing.T) *backlogMesh { - t.Helper() - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - if _, err := inv.AddNode(ctx, "desk"); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "desk", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - wasHeard := releaseHeard - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil - } - t.Cleanup(func() { releaseHeard = wasHeard }) - return b -} - -func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - gateEvery = 0 - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") - if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { - t.Fatalf("sends and judgings: %v", *said) - } - if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { - t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} - -func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - gateEvery = 0 - b.broken["desk"] = true // desk fails what it is sent in the rest's step - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { - t.Fatalf("laptop was never sent: %v", b.sent) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop has no verdict of its own: %+v", p.Release) - } - // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - // app failed on desk: marked, and put back wherever it was sent (ADR 0236). - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { - t.Fatalf("desk was not put back: %v", sent) - } -} - -func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once - release := b.release(t) - judging := release.Release.Gate - if judging == nil || judging.Pending || judging.Passes == 0 { - t.Fatalf("the release is not judging anchor: %+v", judging) - } - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-app-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again what it runs: %v", b.sent) - } - p, err := inv.PlanByID(ctx, "plan-app") - if err != nil { - t.Fatal(err) - } - g := p.Modules["app"].Gate - if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { - t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) - } - if p.Modules["app"].Previous != "c1" { - t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) - } -} - -func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - // The old controller sends anchor app c2 and is replaced before it keeps the send. - made := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if _, err := made(ctx, open, names); err != nil { - return nil, err - } - return nil, context.Canceled - } - advancePlans(ctx, g.open) - sendRollout = made - if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { - t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) - } - last, _, err := g.open.inventory.SentTo(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if len(g.sent) != 1 { - t.Fatalf("the send was made again: %v", g.sent) - } - s := g.plan(t).Modules["app"] - if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { - t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) - } - // What anchor ran before was kept before the send, so a failure still puts it back (§5). - if s.Previous != "c1" || s.Gate.From != "c1" { - t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) - } -} - -func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gate - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} - -func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { - withTheGate(t) - b := aBacklogOnThree(t) - ctx := t.Context() - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - for i := 0; i < 6 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, s := range b.sent { - if !slices.Equal(s, []string{"anchor"}) { - t.Fatalf("sent %v: the rest after a failed first machine", b.sent) - } - } -} - -func TestTheControlNodeIsSentLastInAStep(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) - if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { - t.Fatal(err) - } - said := events(t) - gateEvery = 0 - for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { - t.Fatalf("the order is %v", p.Release.Order) - } - laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") - if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { - t.Fatalf("the control node is not sent last in the rest's step: %v", *said) - } -} - -func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - gateEvery = 0 - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go deleted file mode 100644 index 053ab931..00000000 --- a/cmd/mesh-controller/rest_at_once_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "context" - "slices" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by -// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them — -// a machine already sent what a send is for is not sent again, and its judging goes on from what was read. -// -// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in -// eleven minutes; the new controller its last send started sent the control node again the controller build it -// had just applied, and judged it from its first reading. - -// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others. - -// events records every send and every judging, in order. -func events(t *testing.T) *[]string { - t.Helper() - var said []string - wasSend, wasGather := sendRollout, gatherGateFacts - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - said = append(said, "send "+strings.Join(names, ",")) - return wasSend(ctx, open, names) - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - said = append(said, "judge") - return wasGather(ctx, open, component) - } - t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather }) - return &said -} - -// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second -// was judged to its verdict before the third was sent. - -// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own -// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. - -// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that -// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. - -// A walk's first send that was made and not kept — the controller replaced between the two — is not made again -// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). - -// A release's send kept before it was made, and made, is judged by the controller that takes over without being -// made again (ADR 0353 §8). - -// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits -// for the next step, and one whose every build passed goes now (ADR 0353 §2). -func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { - withTheGate(t) - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, - } - passed := func(module, commit string) bool { return module == "app" && commit == "c2" } - never := func(string) bool { return false } - step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) - } - if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { - t.Fatalf("a machine whose every build passed waits: %v", step) - } -} - -// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step -// carries either to one machine only, while what else passed goes everywhere at once. -func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { - withTheGate(t) - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: hostModule, To: "h2"}}, - "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "dns", To: "d2"}}, - "ace": {{Module: "dns", To: "d2"}}, - } - resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} - alone := func(m string) bool { - if m == "dns" { - return oneAtATime(m, resolver) - } - return oneAtATime(m, catalogue.Manifest{Module: m}) - } - step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, - func(string, string) bool { return true }, alone) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) - } - if oneAtATime("app", catalogue.Manifest{Module: "app"}) { - t.Fatal("a leaf module is held to one machine at a time") - } -} - -// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. - -// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is -// ordered after every other. - -// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put -// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). diff --git a/cmd/mesh-controller/retry_words_test.go b/cmd/mesh-controller/retry_words_test.go deleted file mode 100644 index 4796ff52..00000000 --- a/cmd/mesh-controller/retry_words_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// Issue 1403: a retry whose every module still waits for the bus asked nothing, and does not say "asked" with an empty list. -func TestARetryWhoseModulesAllWaitForTheBusSaysNothingWasAsked(t *testing.T) { - p := inventory.Plan{ID: "plan-9", Tier: 1, Tiers: [][]string{{"a"}, {"b", "c"}}} - note, said := retryWords(p, []string{"b", "c"}, nil, []string{"b", "c"}) - for _, text := range []string{note, said} { - if strings.Contains(text, "asked ;") || strings.Contains(text, "asked ") || strings.HasSuffix(strings.Split(text, ";")[0], "asked") { - t.Errorf("a retry that asked nothing says %q", text) - } - if !strings.Contains(text, "b, c not asked yet: the bus did not answer") { - t.Errorf("it does not say what waits for the bus: %q", text) - } - } - if !strings.Contains(said, "nothing asked yet") || !strings.Contains(note, "nothing could be asked yet") { - t.Errorf("it does not say that nothing was asked: %q / %q", said, note) - } -} - -func TestARetryThatAskedSomeAndLeftSomeForTheBusSaysBoth(t *testing.T) { - p := inventory.Plan{ID: "plan-9", Tier: 0, Tiers: [][]string{{"a", "b"}}} - note, said := retryWords(p, []string{"a", "b"}, []string{"a as build-1"}, []string{"b"}) - if !strings.Contains(note, "a asked again") || strings.Contains(note, "a, b asked again") { - t.Errorf("the note names b as asked again: %q", note) - } - if !strings.Contains(said, "asked a as build-1;") || !strings.Contains(said, "b not asked yet: the bus did not answer") { - t.Errorf("the answer says %q", said) - } -} - -func TestARetryThatAskedEverythingSaysSoAsBefore(t *testing.T) { - p := inventory.Plan{ID: "plan-9", Tier: 0, Tiers: [][]string{{"a"}}} - note, said := retryWords(p, []string{"a"}, []string{"a as build-1"}, nil) - if note != "tier 0 retried by hand: a asked again" || - said != "plan-9 retried at tier 0 of 1: asked a as build-1; the plan goes on from there as any plan does" { - t.Errorf("a retry that asked everything says %q / %q", note, said) - } -} diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go index a8d16bec..d869ba80 100644 --- a/cmd/mesh-controller/rollout.go +++ b/cmd/mesh-controller/rollout.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/nats-io/nats.go/jetstream" "os" "strings" "time" @@ -170,7 +171,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines // The machine that holds the bus seat is the one that would be sent the user list. if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") { state.Holder = n.Name - state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name) + state.AccountsComposed = wasGivenTheUserList(ctx, n.Name, module) } // A module that never speaks needs no credential, so it is not counted as missing one. if !speaksOnTheBus(m) { @@ -196,14 +197,23 @@ func speaksOnTheBus(m catalogue.Manifest) bool { len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0 } -// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user -// list became part of one. +// wasGivenTheUserList says whether the machine holding the bus has a composition of the bus module applied since the +// user list became part of one: its have names a composition at all. // -// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away -// has still been sent it, and this question is about whether the mesh did its part. -func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool { - digest, err := inv.Outstanding(ctx, node) - return err == nil && strings.TrimSpace(digest) != "" +// Read from the node-engine's have rather than asked of the machine: a machine that is away has still been given it, and +// this question is about whether the mesh did its part. +func wasGivenTheUserList(ctx context.Context, node, module string) bool { + bus, err := aBus() + if err != nil { + return false + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return false + } + have, had, err := haveOn(ctx, js, node, module) + return err == nil && had && have.Composed != "" } // notReadyOf is the readiness reasoning, named here so a test can reach it without the command's diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go deleted file mode 100644 index 5818cff8..00000000 --- a/cmd/mesh-controller/rollout_events_test.go +++ /dev/null @@ -1,385 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// Acceptance of the step (novox/hq ADR 0406 §5, §6, acceptance 7; issue 974), live, with the store and a real bus and no -// ticker: a rollout of two tiers advances on the events alone — the plan kept, the machines' reports, a silent key's -// expiry — a machine that reports in time is never named silent and one that never reports is named silent when its -// key expires, and a controller replaced in the middle of it resumes with no tier asked twice and none skipped. -func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - // A quarter of an hour of silence is two seconds here. - wasSilent := applySilentAfter - // Ten seconds here: long enough that laptop's key outlives the controller's replacement under a loaded run. anchor's - // report never races it: anchor answers tier 1 inside its send (reportOnSend below). - applySilentAfter = 10 * time.Second - t.Cleanup(func() { applySilentAfter = wasSilent }) - - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - sentHeard := make(chan *nats.Msg, 32) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - // Every save of a plan is kept, said and looked at by the serving controller's own hook. - saved := &movesSaid{revs: map[string][]int64{}} - wasSaved := inventory.PlanSaved - serving := servingPlanSaved(ctx, checkEvents, open) - inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); serving(i, p) } - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - - // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. - var mu sync.Mutex - var sends [][]string - asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id - digests := 0 - var reportOnSend func(node string) // set below, once a report can be made - anchorReported := make(chan error, 1) - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - mu.Lock() - defer mu.Unlock() - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range []string{"a", "b"} { - builds[m] = current[m].Commit - } - for _, node := range names { - digests++ - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { - return nil, err - } - } - // anchor answers tier 1 as soon as it is sent (issue 1938): its report is in before the silence bound can run, by - // order and not by a race with the bound, which twice named it silent under a loaded run. laptop never reports. - if len(sends) == 2 && reportOnSend != nil { - for _, node := range names { - if node == "anchor" { - go reportOnSend(node) - } - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - register := func(m, commit string, asked time.Time) { - t.Helper() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - record := func(id, m, commit string, at time.Time) { - t.Helper() - b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - old := time.Now().Add(-2 * time.Hour) - for _, m := range []string{"a", "b"} { - record("build-"+m+"-1", m, "c1", old) - register(m, "c1", old) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { - t.Fatal(err) - } - } - record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) - register("a", "c2", time.Now().Add(-time.Minute)) - - // A machine's report on what it was sent, heard as the controller hears it. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - seq := int64(0) - reports := func(node string) { - t.Helper() - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - t.Fatalf("%s was sent nothing: %v", node, err) - } - seq++ - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": seq}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - // tell is a report heard as the controller hears it, for a goroutine: its error is handed back, not fatal. - tell := func(node string) error { - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - return fmt.Errorf("%s was sent nothing: %v", node, err) - } - mu.Lock() - seq++ - n := seq - mu.Unlock() - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": n}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - return err - } - _, err = listener.Heard(ctx, report) - return err - } - reportOnSend = func(node string) { anchorReported <- tell(node) } - heldKeys := func() map[silentWait]time.Time { - c := silentClockOf() - if c == nil { - return nil - } - held, err := c.Held(ctx) - if err != nil { - t.Fatal(err) - } - return held - } - plan := func() inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, "plan-974") - if err != nil { - t.Fatal(err) - } - return p - } - - // Tier 1's module b has its key unbuilt while the first controller runs, so the plan stands at tier 1 as it did while - // its build ran; with the key done at once, the same look would take it and send tier 1 as well. - heldB := holdKeys(t, "b") - // The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything. - first, stopFirst := context.WithCancel(ctx) - defer stopFirst() - waitFirst := startRolloutEvents(first, open, js) - built := time.Now().UTC() - p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, - "b": {}, - }} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - eventually(t, "tier 0 sent to both machines and tier 1 taken from its key, on the save alone", func() bool { - mu.Lock() - defer mu.Unlock() - // b is taken from its key; the walk carries a commit of novox/mesh-catalog only, and b is registered from - // novox/b, so the key is read at b's own trunk value and KeyAt stays empty (Plan.CommitOn names no other - // repository's commit). - b := plan().Modules["b"] - return len(sends) == 1 && len(*asked) == 0 && plan().Tier == 1 && b != nil && b.Key && b.KeyAt == "" - }) - eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 }) - - // The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody. - stopFirst() - waitFirst() - time.Sleep(300 * time.Millisecond) - reports("anchor") - - heldB.Store(false) // b's key is built by the time the new controller looks - second, stopSecond := context.WithCancel(ctx) - defer stopSecond() - waitSecond := startRolloutEvents(second, open, js) - // Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338). - t.Cleanup(func() { stopSecond(); waitSecond() }) - // Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire. - eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool { - h := heldKeys() - _, anchor := h[silentWait{Plan: "plan-974", Tier: 0, Node: "anchor"}] - return !anchor - }) - - // tier 1's module is taken from its key by the new controller's look. - eventually(t, "tier 1 sent to both machines", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 2 - }) - // anchor reported on tier 1 inside its send. - select { - case err := <-anchorReported: - if err != nil { - t.Fatalf("anchor's report on tier 1: %v", err) - } - case <-time.After(15 * time.Second): - t.Fatal("anchor never reported on tier 1") - } - - // laptop never reports: its key expires and the new controller names it silent, which ends the rollout. - eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool { - p := plan() - return p.State == inventory.PlanDone - }) - p = plan() - if !strings.Contains(p.Note, "laptop") || strings.Contains(p.Note, "anchor") || !strings.Contains(p.Note, "said nothing") { - t.Errorf("the plan ended with the note %q; want laptop named silent and not anchor", p.Note) - } - eventually(t, "no key left on the bus", func() bool { return len(heldKeys()) == 0 }) - - // No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent - // once to both machines, and said as sent once for each. - mu.Lock() - if len(*asked) != 1 || !strings.HasPrefix((*asked)[0], "novox/b#key-") { // b is registered from novox/b - t.Errorf("taken from keys = %v; want tier 1's module taken once", *asked) - } - if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" { - t.Errorf("sends = %v; want both machines sent each tier once", sends) - } - mu.Unlock() - said := map[string]int{} - for deadline := time.After(5 * time.Second); len(said) < 4; { - select { - case m := <-sentHeard: - var body planSentBody - if err := json.Unmarshal(m.Data, &body); err != nil { - t.Fatal(err) - } - said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++ - case <-deadline: - t.Fatalf("sent was said for %v; want tiers 0 and 1 to both machines", said) - } - } - time.Sleep(300 * time.Millisecond) - for len(sentHeard) > 0 { - var body planSentBody - _ = json.Unmarshal((<-sentHeard).Data, &body) - said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++ - } - for k, n := range said { - if n != 1 { - t.Errorf("sent for %s was said %d times", k, n) - } - } - // And every move of the plan was said as plan-moved. - saved.noUnsaid(t, open) -} - -// Acceptance of the verb path (issue 974, review 1), live, with the store and a bus: a walk the delivery's owner lets -// go by the verb `deliver` after the operator's approval is asked its first tier with no other event. The verb runs as a -// child process, whose saves the serving controller does not hear; the handler asks the look as the verb ends. -func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - asked := asksRecorded(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - cut := time.Now().UTC() - waiting := inventory.Plan{ID: "plan-974-deliver", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", - Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, - Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - - // The serving controller, as serve starts it. - wasSaved := inventory.PlanSaved - inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - started, stop := context.WithCancel(ctx) - defer stop() - wait := startRolloutEvents(started, open, js) - time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing - - // The verb as its child process runs it: the delivery command, whose saves are not the serving controller's. - wasRun := runTheVerb - runTheVerb = func(ctx context.Context, argv []string) (verbAnswer, error) { - was := inventory.PlanSaved - inventory.PlanSaved = func(*inventory.Inventory, inventory.Plan) {} - defer func() { inventory.PlanSaved = was }() - return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}) - } - t.Cleanup(func() { runTheVerb = wasRun }) - // Registered after every variable cleanup above, so it runs before them (last in, first out): the look at the plans - // has returned when checkEvents, inventory.PlanSaved and runTheVerb are restored (issue 1338). - t.Cleanup(func() { stop(); wait() }) - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - if _, err := handlers["deliver"](ctx, []byte(`{"plan":"`+waiting.ID+`","why":"its turn"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "the walk's first tier asked, with no event but the verb's end", func() bool { - p, err := open.inventory.PlanByID(ctx, waiting.ID) - return err == nil && p.Modules["app"] != nil && p.Modules["app"].Build != "" && len(*asked) == 1 - }) -} diff --git a/cmd/mesh-controller/rollout_first_test.go b/cmd/mesh-controller/rollout_first_test.go deleted file mode 100644 index 02996756..00000000 --- a/cmd/mesh-controller/rollout_first_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only -// once that machine has reported it applied; a module whose policy says together goes everywhere at -// once, as before. -func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) { - running := []string{"novox", "ace", "g14"} - sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - now := sentAt.Add(5 * time.Minute) - bound := 30 * time.Minute - after := sentAt.Add(time.Minute) - next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep { - return nextRollout(s, running, together, reports, now, bound) - } - - // Together: every machine at once. - if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first { - t.Fatalf("a together policy did not send every machine at once: %+v", step) - } - - // Otherwise the first by name, alone, when none has reported lately. - step := next(inventory.PlanModule{}, running, false, nil) - if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) { - t.Fatalf("the first send was %+v, wanted ace alone", step) - } - - state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt} - report := func(outcome string, current bool) []inventory.Reported { - return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current}, - {Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}} - } - - // No report yet, or one about an older declaration than it was last sent: wait. - for what, reports := range map[string][]inventory.Reported{ - "no report": nil, - "a report about older": report(inventory.OutcomeApplied, false), - } { - step := next(state, running, false, reports) - if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" { - t.Errorf("%s: %+v, wanted to wait for ace", what, step) - } - } - - // Applied what it was last sent: the rest, and only the rest. - step = next(state, running, false, report(inventory.OutcomeApplied, true)) - if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) { - t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step) - } - - // Failed or refused: stop, the rest untouched. - for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} { - step := next(state, running, false, report(outcome, true)) - if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) || - !reflect.DeepEqual(step.rest, []string{"novox", "g14"}) { - t.Errorf("a first machine that %s it: %+v", outcome, step) - } - } - - // The machine holding the bus went with the first send: the rest wait for it too, and it is - // not sent again. - both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt} - half := report(inventory.OutcomeApplied, true) - if step := next(both, running, false, half); step.waiting != "novox" { - t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step) - } - all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true}) - if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) { - t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step) - } - - // One machine, or none: nothing is waited for that cannot come. - if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first { - t.Fatalf("a module nothing runs was sent: %+v", step) - } - if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" { - t.Fatalf("a module on one machine waited for more: %+v", step) - } -} - -// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there, -// naming the machine and the bound; the rest are left alone. -func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) { - sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt} - step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute) - if !strings.Contains(step.failed, "ace did not report it applied within 30m") || - !reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 { - t.Fatalf("a silent first machine: %+v", step) - } -} - -// The first machine is the first by name among those heard from lately: a laptop that is away is -// not the one the rest wait on. When none has been heard from, the first by name. -func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) { - now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour) - reports := []inventory.Reported{ - {Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately}, - } - step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute) - if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) { - t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step) - } -} - -// An announced move of a module an open plan is still rolling out is left to the plan: sending it -// here as well put the bundle on every machine at once (novox/hq issue 249). -func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) { - sent := time.Now() - plans := []inventory.Plan{ - {ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}}, - {ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}}, - } - if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" { - t.Fatalf("a module the plan is rolling out was not left to it: %q", got) - } - for _, m := range []string{"gitea", "keycloak"} { - if got := rolledOutByAPlan(plans, m); got != "" { - t.Errorf("%s, which no plan will send, was left to %s", m, got) - } - } -} - -// A plan that ends without sending what it built says so, with the remedy; a module whose rollout -// stopped at its first machine is not among them. -func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) { - at := time.Now() - p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1", - Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built"}, - "stopped": {State: "built", First: []string{"ace"}, FirstAt: &at}, - "sent": {State: "built", SentAt: &at}, - "notes": {State: "built"}, - "later": {}, - }} - rollsOut := func(m string) bool { return m != "notes" } - sayUnsent(&p, rollsOut) - sayUnsent(&p, rollsOut) - if p.Note != "closed by hand at tier 1; built and never sent: agent — `upgrade release-backlog --why ` releases them" { - t.Fatalf("the note reads %q", p.Note) - } -} diff --git a/cmd/mesh-controller/rollout_silent_bus.go b/cmd/mesh-controller/rollout_silent_bus.go deleted file mode 100644 index 28a46b9b..00000000 --- a/cmd/mesh-controller/rollout_silent_bus.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" -) - -// A rollout's wait for a machine is a key on the bus (novox/hq ADR 0406 §5, issue 974): each send to a machine puts -// a key `..` that lives as long as the machine may stay silent (ApplySilentAfter); the machine's -// report on that send deletes it, so only a missing report lets it expire; and the expiry, heard by a watcher, -// ends the wait for that machine, which the plan then names silent. No timer looks every so often whether a -// machine has reported. - -// silentWait is one machine's wait on one send of a plan's tier. -type silentWait struct { - Plan string - Tier int - Node string -} - -// silentKey is a wait's key. A plan id may hold a dot, so it is read from the right (parseSilentKey). -func silentKey(w silentWait) string { return w.Plan + "." + strconv.Itoa(w.Tier) + "." + w.Node } - -// parseSilentKey reads a key back: the node is the last token, the tier the one before, the plan what is left. -func parseSilentKey(key string) (silentWait, bool) { - last := strings.LastIndexByte(key, '.') - if last < 1 || last == len(key)-1 { - return silentWait{}, false - } - rest, node := key[:last], key[last+1:] - mid := strings.LastIndexByte(rest, '.') - if mid < 1 || mid == len(rest)-1 { - return silentWait{}, false - } - tier, err := strconv.Atoi(rest[mid+1:]) - if err != nil || tier < 0 { - return silentWait{}, false - } - return silentWait{Plan: rest[:mid], Tier: tier, Node: node}, true -} - -// silentClock is the bus's side of the wait for a machine's report. -type silentClock interface { - // Arm puts a wait's key, to live ttl: made again if the machine is sent again. The key holds when the send was. - Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error - // Held is every wait that has a key now, with when its send was. - Held(ctx context.Context) (map[silentWait]time.Time, error) - // Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller. - Reported(ctx context.Context, w silentWait) error -} - -// busSilent is the clock on the controller's bucket. -type busSilent struct { - kv jetstream.KeyValue -} - -// newBusSilent opens the bucket and starts the watcher, which names each wait whose key expired until the context -// ends. Only expiry wakes it: the marker of an expiry is a purge, and a key a report deleted, or one Arm made again, -// leaves a delete (the vendored client reads the server's MaxAge marker as a purge, kv.go:971). -func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(silentWait)) (*busSilent, error) { - kv, err := api.KeyValue(ctx, broker.RolloutSilentBucket) - if err != nil { - return nil, fmt.Errorf("the rollout's silent keys' bucket: %w", err) - } - w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) - if err != nil { - return nil, fmt.Errorf("watching the rollout's silent keys: %w", err) - } - go func() { - defer w.Stop() - for { - select { - case <-ctx.Done(): - return - case e, ok := <-w.Updates(): - if !ok { - return - } - if e == nil || e.Operation() != jetstream.KeyValuePurge { - continue // a put, or a delete: a report's, or a send made again - } - if wait, ok := parseSilentKey(e.Key()); ok && silent != nil { - silent(wait) - } - } - } - }() - return &busSilent{kv: kv}, nil -} - -func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error { - key := silentKey(w) - // Made again, not changed: a key's time to live is set when it is made. - if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - _, err := s.kv.Create(ctx, key, []byte(sent.UTC().Format(time.RFC3339Nano)), jetstream.KeyTTL(leastSecond(ttl))) - return err -} - -func (s *busSilent) Reported(ctx context.Context, w silentWait) error { - if err := s.kv.Delete(ctx, silentKey(w)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - return nil -} - -// leastSecond rounds a time to live up to a whole second, at least one: the server rounds a key's time down to -// whole seconds, so a fraction would close the key before the time it stands for. -func leastSecond(d time.Duration) time.Duration { - if d < time.Second { - return time.Second - } - return (d + time.Second - 1) / time.Second * time.Second -} - -func (s *busSilent) Held(ctx context.Context) (map[silentWait]time.Time, error) { - keys, err := s.kv.Keys(ctx) - if errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil - } - if err != nil { - return nil, err - } - held := map[silentWait]time.Time{} - for _, key := range keys { - w, ok := parseSilentKey(key) - if !ok { - continue - } - e, err := s.kv.Get(ctx, key) - if err != nil { - if errors.Is(err, jetstream.ErrKeyNotFound) { - continue - } - return nil, err - } - at, _ := time.Parse(time.RFC3339Nano, string(e.Value())) - held[w] = at - } - return held, nil -} diff --git a/cmd/mesh-controller/rollout_silent_bus_integration_test.go b/cmd/mesh-controller/rollout_silent_bus_integration_test.go deleted file mode 100644 index 571c73fe..00000000 --- a/cmd/mesh-controller/rollout_silent_bus_integration_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" -) - -func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan silentWait, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w }) - if err != nil { - t.Fatal(err) - } - return s, heard -} - -func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { // 1157/AC3: the send keys stay until release 6 (ADR 0433 §5) - s, heard := silentOnABus(t) - want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} - if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { - t.Fatalf("heard %+v (%v); want %+v", got, ok, want) - } -} - -func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} - if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := s.Reported(ctx, w); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok { - t.Fatalf("%+v was named silent after it reported", got) - } - // Reporting twice, or on a send that put no key, is no error. - if err := s.Reported(ctx, w); err != nil { - t.Fatalf("a second report: %v", err) - } -} - -func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("%+v expired at the first send's time, not the second's", got) - } - if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w { - t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w) - } -} diff --git a/cmd/mesh-controller/rollout_silent_bus_test.go b/cmd/mesh-controller/rollout_silent_bus_test.go deleted file mode 100644 index aaf3aa85..00000000 --- a/cmd/mesh-controller/rollout_silent_bus_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "testing" - "time" -) - -// silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire. - -func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) { - t.Helper() - select { - case w := <-heard: - return w, true - case <-time.After(within): - return silentWait{}, false - } -} - -// A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5): -// the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur. - -// A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry. - -// A send made again to a machine puts its key again with the full time, and only the last send's key can expire. - -func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) { - for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} { - got, ok := parseSilentKey(silentKey(w)) - if !ok || got != w { - t.Errorf("%+v became %+v (%v)", w, got, ok) - } - } - for _, bad := range []string{"", "plan", "plan.x.ace", "plan..ace", ".1.ace"} { - if _, ok := parseSilentKey(bad); ok { - t.Errorf("%q was read as a wait", bad) - } - } -} diff --git a/cmd/mesh-controller/rotate.go b/cmd/mesh-controller/rotate.go index 3ecdc25d..a05fd04c 100644 --- a/cmd/mesh-controller/rotate.go +++ b/cmd/mesh-controller/rotate.go @@ -142,20 +142,19 @@ func rotateCommand(ctx context.Context, args []string) error { } } - // **Both ends, in one send.** There is a window either way — a role's password changes on the - // provider and the file changes on the consumer, and they cannot be simultaneous — so the - // honest thing is to make it as short as the broker allows and to never leave it open across - // a command boundary, where it depends on somebody's memory. - fmt.Printf("\nsending to both ends:\n") - if err := sendTo(ctx, open, machines); err != nil { + // **Both ends, composed together** (novox/hq ADR 0539 clause 5). There is a window either way — a role's + // password changes on the provider and the file changes on the consumer, and they cannot be simultaneous — + // so both ends' compositions are written in one act, never left across a command boundary. + fmt.Printf("\ncomposing both ends:\n") + if _, err := composeModules(ctx, open, machines); err != nil { return fmt.Errorf( - "%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+ - "Nothing on those machines has changed yet, so what is running keeps working "+ - "until the provider next applies. Fix the cause and run `push --behind`", err) + "%w\n\nThe old credentials are gone from the mesh and the new compositions are not all written. "+ + "Nothing on those machines has changed yet, so what is running keeps working until each "+ + "node applies the composition its want names. Fix the cause and rotate again", err) } - fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+ - "changed cannot authenticate — `status` says who is still behind\n", len(machines)) + fmt.Printf("\n%d machine(s) composed. Until both ends have applied what their wants name, a consumer whose "+ + "password changed cannot authenticate — the delivery view says each machine's want and have\n", len(machines)) return nil } diff --git a/cmd/mesh-controller/router_ids_integration_test.go b/cmd/mesh-controller/router_ids_integration_test.go deleted file mode 100644 index 2f238fe9..00000000 --- a/cmd/mesh-controller/router_ids_integration_test.go +++ /dev/null @@ -1,52 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "testing" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/testbus" -) - -// Issue 1336: the catch-up lists the ids of an asker's asks in the router's bucket. The key lister stops its own watch when the last -// key is read, and a second stop answers "nats: invalid subscription"; read as a failure, it made every warrant unreadable on novox -// (2026-10-20, 110 gaps named, none planned). -func TestTheAskIDsOfAnAskerAreReadFromTheRoutersBucketWithoutAFalseFailure(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - ctx := t.Context() - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "router_asks_test"}) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), "router_asks_test") }) - for _, key := range []string{"mesh-delivery.merge-378-b9104454f9a5-1", "mesh-delivery.merge-377-aaaaaaaaaaaa-1", - "mesh-delivery-teller.merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "keycloak.something"} { - if _, err := kv.Put(ctx, key, []byte(`{}`)); err != nil { - t.Fatal(err) - } - } - got, err := askIDsOf(ctx, kv, "mesh-delivery") - if err != nil { - t.Fatalf("the ids of an asker's asks could not be read: %v", err) - } - slices.Sort(got) - if want := []string{"merge-377-aaaaaaaaaaaa-1", "merge-378-b9104454f9a5-1"}; !slices.Equal(got, want) { - t.Fatalf("read %v, want %v: the teller's and another module's asks are not mesh-delivery's", got, want) - } - if none, err := askIDsOf(ctx, kv, "nobody"); err != nil || len(none) != 0 { - t.Fatalf("an asker with no asks read as %v, %v", none, err) - } -} diff --git a/cmd/mesh-controller/same_program.go b/cmd/mesh-controller/same_program.go deleted file mode 100644 index b044536e..00000000 --- a/cmd/mesh-controller/same_program.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "context" - "fmt" - "slices" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A module moves only when its source changes** (novox/hq issue 721; from R4e by its fingerprint, ADR 0453 §6, ADR -// 0499; issue 1851, A9). A walk is cut without a module whose fingerprint at the commit the walk carries is the -// fingerprint of the build the mesh holds of it, and says so among the modules it left unchanged: a merge of a file no -// module reads moves nothing. A build the mesh holds of the old form (build-agent's `src1:`) has no fingerprint to -// compare: the module stays in the walk, and the rule of the standing build (equal digests, no move) judges it once its -// key is taken. - -// sameByFingerprint are the modules whose fingerprint now (`src2:`, the indexer's trunk value at the walk's commit) is -// the fingerprint of the build the mesh holds, each with why; one whose either is unknown, or of the old form, is not. -func sameByFingerprint(now, held map[string]string) map[string]string { - out := map[string]string{} - for m, fp := range now { - if strings.HasPrefix(fp, "src2:") && held[m] == fp { - out[m] = fmt.Sprintf("unchanged: its fingerprint %s is the build the mesh holds", short(fp)) - } - } - if len(out) == 0 { - return nil - } - return out -} - -// heldFingerprint is the source fingerprint of the build the mesh holds of a module: the newest build of the commit -// it is registered at. -func heldFingerprint(ctx context.Context, inv *inventory.Inventory, module string) (string, error) { - src, err := inv.SourceOf(ctx, module) - if err != nil { - return "", err - } - fps, err := inv.BuildSourceFingerprints(ctx, module) - if err != nil { - return "", err - } - for commit, fp := range fps { - if sameCommitID(commit, src.BuiltFrom) { - return fp, nil - } - } - return "", nil -} - -// leaveSameAsRunning takes from a walk being cut every module whose fingerprint is unchanged (sameByFingerprint), and -// every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read -// leaves the walk as it was**, said. -func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string, - edges []inventory.Edge) map[string]string { - if len(plan.Modules) == 0 { - return nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - fmt.Printf(" the catalogue could not be read, so every module moves: %v\n", err) - // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before - return nil - } - keys, closeKeys, err := openPlanKeys() - if err != nil { - fmt.Printf(" the indexer's fingerprints could not be read, so every module moves: %v\n", err) - // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before - return nil - } - defer closeKeys() - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - now, held := map[string]string{}, map[string]string{} - for m := range plan.Modules { - e, known := byName[m] - if !known { - continue - } - v, err := keys.ModuleAt(ctx, m, plan.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref))) - if err != nil || v.FP2 == "" { - continue // not written yet, or none: the module moves, and its key is waited for in the walk - } - fp, err := heldFingerprint(ctx, inv, m) - if err != nil { - continue - } - now[m], held[m] = v.FP2, fp - } - same := withDependentsOf(sameByFingerprint(now, held), plan, moved, edges) - if len(same) == 0 { - return nil - } - names := make([]string, 0, len(same)) - for n := range same { - names = append(names, n) - } - sort.Strings(names) - withhold(plan, names, nil) - for _, n := range names { - fmt.Printf(" %s left out: %s\n", n, same[n]) - } - return same -} - -// withDependentsOf adds to the modules found the same every module of the walk that is in it only as a dependent of -// one of them (novox/hq ADR 0267 rule 3): **a dependent is left out when its base comes out the same**, since the -// base it stands on stays the build the mesh holds. A module a merge moves itself, or a dependent of a base that -// still moves, stays. moved are the modules the walk's merges move; edges the graph the walk was planned along. -func withDependentsOf(same map[string]string, plan *inventory.Plan, moved []string, edges []inventory.Edge) map[string]string { - if len(same) == 0 { - return same - } - var still []string - for _, n := range moved { - if same[n] == "" { - still = append(still, n) - } - } - keep := map[string]bool{} - for _, n := range reachableFrom(still, edges) { - keep[n] = true - } - // Until nothing more is found: a dependent of a dependent left out is left out too. - for grew := true; grew; { - grew = false - for n := range plan.Modules { - if keep[n] || same[n] != "" { - continue - } - var bases []string - for _, e := range edges { - ordering := e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || - e.Kind == inventory.EdgePackages || e.Kind == edgeChangeOrder - if !ordering && e.From == n && same[e.To] != "" && !slices.Contains(bases, e.To) { - bases = append(bases, e.To) - } - } - sort.Strings(bases) - if len(bases) == 0 { - // In the walk for no base found the same: whatever brought it in is not said here, so it moves. - continue - } - same[n] = "unchanged: it is in the walk only for " + strings.Join(bases, ", ") + - ", which came out the same as the build the mesh holds (ADR 0267 rule 3)" - grew = true - } - } - return same -} diff --git a/cmd/mesh-controller/same_program_integration_test.go b/cmd/mesh-controller/same_program_integration_test.go deleted file mode 100644 index 0a4f5131..00000000 --- a/cmd/mesh-controller/same_program_integration_test.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// R4e A9 (ADR 0453 §6, ADR 0499; issue 1851), against the store and a real bus: a merge of a file no module reads moves -// nothing. Its module's fingerprint at the merge, as the indexer wrote it, is the fingerprint of the build the mesh holds, -// so the walk is cut done with nothing to walk, and its record names the module unchanged and why — the walk -// mesh-delivery reads as delivered, never stranded; no build is asked. -func TestAWalkWhoseEveryFingerprintIsUnchangedIsDone(t *testing.T) { - open := windowed(t) - endsItsLine(t) - noBuildAsked(t) - b := aBuilderOnTheBus(t) - ctx := t.Context() - inv := open.inventory - fp := fpOf('5') - if err := inv.RecordBuild(ctx, inventory.Build{ID: "key-held", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "ace", SourceFingerprint: fp, - Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, - inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { - t.Fatal(err) - } - code := repoMerge("one", "one1aaaaaaaa", t0) - b.indexed(t, "one", "1", "novox/one", "", code.Commit, fp) - hear(t, open, code, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "is the build the mesh holds") { - t.Fatalf("the walk does not say one's fingerprint is unchanged: %+v", w.Delivery) - } -} diff --git a/cmd/mesh-controller/same_program_test.go b/cmd/mesh-controller/same_program_test.go deleted file mode 100644 index 39b4a467..00000000 --- a/cmd/mesh-controller/same_program_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange is novox/hq issue 721 on fingerprints (R4e A9, ADR 0453 §6): -// mesh-controller #335 changed one sentence that mesh-builder never reaches; the indexer's fingerprint of build-agent at -// the merge is the fingerprint of the build the mesh holds, so the walk moves mesh-controller alone and says build-agent -// is unchanged. A held build of the old form, or a fingerprint not written, leaves the module in the walk. -func TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange(t *testing.T) { - a, b := "src2:"+strings.Repeat("a", 64), "src2:"+strings.Repeat("b", 64) - now := map[string]string{"build-agent": a, "mesh-controller": b, "old": a, "unwritten": ""} - held := map[string]string{"build-agent": a, "mesh-controller": a, "old": "src1:" + strings.Repeat("a", 64)} - same := sameByFingerprint(now, held) - if len(same) != 1 || !strings.Contains(same["build-agent"], "is the build the mesh holds") { - t.Fatalf("the walk leaves out %v, want build-agent alone", same) - } - p := &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}}, - Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}}} - withhold(p, []string{"build-agent"}, nil) - if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" { - t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers) - } -} - -// TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk -// for its base; when the base comes out the same as the build the mesh holds, the dependent stands on that build -// still, and leaves the walk with it — and so does a dependent of that dependent. One a merge moves itself stays. -func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) { - edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeDeclared}, - {From: "app2", To: "app", Kind: inventory.EdgeDeclared}, {From: "own", To: "base", Kind: inventory.EdgeDeclared}} - plan := &inventory.Plan{Modules: map[string]*inventory.PlanModule{"base": {}, "app": {}, "app2": {}, "own": {}}} - same := withDependentsOf(map[string]string{"base": "unchanged: its program is the same"}, plan, - []string{"base", "own"}, edges) - if !strings.Contains(same["app"], "only for base") || !strings.Contains(same["app2"], "only for app") { - t.Fatalf("the dependents of a base that came out the same are not left out: %v", same) - } - if _, left := same["own"]; left { - t.Fatalf("a module its merge moves itself is left out for its base: %v", same) - } - // A dependent of a base that still moves stays. - edges = append(edges, inventory.Edge{From: "app", To: "own", Kind: inventory.EdgeDeclared}) - same = withDependentsOf(map[string]string{"base": "unchanged"}, plan, []string{"base", "own"}, edges) - if _, left := same["app"]; left { - t.Fatalf("a dependent of a base that still moves is left out: %v", same) - } -} - -// Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one -// module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the -// module unchanged and why — the walk mesh-delivery reads as delivered, never stranded. diff --git a/cmd/mesh-controller/same_source_test.go b/cmd/mesh-controller/same_source_test.go deleted file mode 100644 index b88fd010..00000000 --- a/cmd/mesh-controller/same_source_test.go +++ /dev/null @@ -1,212 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A build whose source is unchanged is never a move (novox/hq issue 280): an image is not -// byte-reproducible, so two builds of one source make two digests, and the rule that a rebuild with -// identical artifacts is no move (ADR 0236) never held for one. - -// anImageBuild is a build outcome of an image module: its manifest names the image by the digest made. -func anImageBuild(t *testing.T, module, id, commit, digest, source string, asked time.Time) link.BuildResult { - t.Helper() - image := "registry.invalid:5000/" + module + "/server@sha256:" + digest - manifest, _ := json.Marshal(map[string]any{"module": module, "version": "1", - "resources": []any{map[string]any{"id": "svc", "type": "container", "name": module, "image": image}}}) - return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module, - On: "anchor", Module: module, Commit: commit, Manifest: manifest, SourceFingerprint: source, - Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: image}}, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} -} - -// shelfNames is whether the module the mesh holds names this digest. -func shelfNames(t *testing.T, inv *inventory.Inventory, module, digest string) bool { - t.Helper() - shelf, err := inv.Catalogue(t.Context()) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(shelf[module]) - return strings.Contains(string(raw), digest) -} - -// A rebuild of an unchanged source is registered with the artifacts of the build it was first made -// as: no machine is sent a new digest, a module standing on it is handed the same base, and the two -// builds are one to every rule that asks whether a send moves something. A real source change moves. -func TestARebuildOfAnUnchangedSourceKeepsItsArtifacts(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - for i, b := range []link.BuildResult{ - anImageBuild(t, "app", "", "c1aaaaaa", strings.Repeat("a", 64), "src1:same", start), - // Another module's merge rebuilt it: a new commit, a new image digest, the same source. - anImageBuild(t, "app", "", "c2bbbbbb", strings.Repeat("b", 64), "src1:same", start.Add(time.Minute)), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatalf("build %d: %v", i, err) - } - } - if !shelfNames(t, inv, "app", strings.Repeat("a", 64)) || shelfNames(t, inv, "app", strings.Repeat("b", 64)) { - t.Fatal("a rebuild of an unchanged source registered the digest it made, not the one the mesh holds") - } - if src, err := inv.SourceOf(ctx, "app"); err != nil || src.BuiltFrom != "c2bbbbbb" { - t.Fatalf("the module is not at the commit it was rebuilt from: %+v %v", src, err) - } - held, err := inv.Held(ctx) - if err != nil || !strings.HasSuffix(held["app/server"], strings.Repeat("a", 64)) { - t.Fatalf("a module standing on it is handed %q, not the build the mesh holds (%v)", held["app/server"], err) - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - if !f.identical("app", "c1aaaaaa", "c2bbbbbb") { - t.Fatal("two builds of one source are not one build") - } - - // A real change to the source moves: its own digest registered, and not identical. - if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c3cccccc", strings.Repeat("c", 64), "src1:changed", - start.Add(2*time.Minute))); err != nil { - t.Fatal(err) - } - if !shelfNames(t, inv, "app", strings.Repeat("c", 64)) { - t.Fatal("a changed source did not register what it made") - } - if f, _ = readMoveFacts(ctx, inv); f.identical("app", "c2bbbbbb", "c3cccccc") { - t.Fatal("a changed source is read as the same build") - } - // And a builder that says no fingerprint is told apart by its artifacts alone, as before. - if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c4dddddd", strings.Repeat("d", 64), "", - start.Add(3*time.Minute))); err != nil { - t.Fatal(err) - } - if !shelfNames(t, inv, "app", strings.Repeat("d", 64)) { - t.Fatal("a build with no fingerprint did not register what it made") - } -} - -// The bus rebuilt for another module's merge, its source untouched and its image digest new: no move — -// no push to its machine is held, no bus step is demanded, and `bus upgrade` has nothing to do. A real -// change to the bus's source is the planned step again. -func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - bus := func(id, commit, digest, source string, at time.Time) link.BuildResult { - b := anImageBuild(t, "nats", id, commit, digest, source, at) - manifest, _ := json.Marshal(map[string]any{"module": "nats", "version": "2", - "provides": []any{map[string]any{"name": "mesh-bus"}}, - "resources": []any{map[string]any{"id": "svc", "type": "container", "name": "nats", - "image": b.Made[0].Reference}}}) - b.Manifest = manifest - return b - } - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, bus("", "n1111111", strings.Repeat("a", 64), "src1:bus", start))); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1111111"}); err != nil { - t.Fatal(err) - } - // The wide rebuild: a new commit, a new image digest, the same source. - if _, _, err := takeIn(ctx, inv, bus("", "n2222222", strings.Repeat("b", 64), "src1:bus", start.Add(time.Minute))); err != nil { - t.Fatal(err) - } - if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { - t.Fatalf("a bus rebuilt from an unchanged source held its machine: %v %v", held, err) - } - wasSnapshot := takeBusSnapshot - t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) - takeBusSnapshot = func(context.Context, string, string) (string, error) { - return "", errors.New("no snapshot is taken for a bus step nobody needs") - } - var sent [][]string - wasSend := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - if err := busCommand(ctx, []string{"upgrade", "--why", "nothing changed", "--reversible"}); err != nil || len(sent) != 0 { - t.Fatalf("a bus step ran for a bus whose source is unchanged: %v, sent %v", err, sent) - } - if !shelfNames(t, inv, "nats", strings.Repeat("a", 64)) { - t.Fatal("the bus the mesh holds is not the image its machine runs") - } - - // A real change to the bus's source: the planned step. - if _, _, err := takeIn(ctx, inv, bus("", "n3333333", strings.Repeat("c", 64), "src1:bus-2.12", start.Add(2*time.Minute))); err != nil { - t.Fatal(err) - } - held, err := busHeld(ctx, inv, []string{"anchor"}) - if err != nil || !strings.Contains(held["anchor"], "planned step") { - t.Fatalf("a changed bus did not demand its planned step: %v %v", held, err) - } -} - -// A plan's build made from the source every machine running the module runs is not sent and not -// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before. -func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) { - judgeLeavesAsTheCore(t) - for _, tc := range []struct { - name string - source string - sent [][]string - }{ - {"unchanged", "src1:app", nil}, - {"changed", "src1:app-changed", [][]string{{"anchor"}}}, - } { - t.Run(tc.name, func(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // What anchor and laptop run (c1) was made from src1:app; the plan's build of c2 from the source - // the case says. - for _, b := range []inventory.Build{ - {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", - Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, - {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: tc.source, - Asked: time.Now(), At: time.Now()}, - } { - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - p := g.plan(t) - p.Modules["app"].Build = "build-2s" - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if !reflect.DeepEqual(g.sent, tc.sent) { - t.Fatalf("sent %v, want %v", g.sent, tc.sent) - } - p = g.plan(t) - s := p.Modules["app"] - if tc.sent == nil && (s.SentAt == nil || s.Gate != nil || !strings.Contains(s.Why, "no move")) { - t.Fatalf("an unchanged source was not read as no move: %+v", s) - } - if tc.sent != nil && (s.Gate == nil || len(s.First) == 0) { - t.Fatalf("a changed source was not sent to its first machine, gated: %+v", s) - } - }) - } -} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 87350b89..ae2e4643 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -58,6 +58,9 @@ func argvFor(verb string, args map[string]any) ([]string, error) { if removedVerbs[verb] { return nil, removedByName(verb) } + if why, removed := removedInRelease6(verb); removed { + return nil, &removedCommand{msg: fmt.Sprintf("`%s` no longer exists: %s", verb, why)} + } a, err := readArguments(verb, args) if err != nil { return nil, err @@ -93,10 +96,22 @@ type verbArguments struct { } // removedVerbs are the controller seat's verbs R4e removed, kept in its table with no input so a caller is answered -// by name (ADR 0501 §5, ADR 0503 §3, ADR 0450 §8). +// by name (ADR 0501 §5, ADR 0503 §3, ADR 0450 §8). The verbs release 6 removed are catalogue.RemovedControllerVerbs. var removedVerbs = map[string]bool{"build": true, "rebuild": true, "forget-checks": true, "delivery-plan": true, "delivery-check": true, "queue": true, "cancel": true, "clear": true, "kill": true, "replay": true, "pause": true, "resume": true} +// removedVerb says a verb is one the seat answers by name only: removed in release 4e or in release 6. +func removedVerb(name string) bool { + _, r6 := removedInRelease6(name) + return removedVerbs[name] || r6 +} + +// removedInRelease6 says a verb is one release 6 removed (ADR 0539, ADR 0429 §8): refused by name, with its replacement. +func removedInRelease6(verb string) (string, bool) { + why, removed := catalogue.RemovedControllerVerbs[verb] + return why, removed +} + // controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the // arguments are checked against the table compiled beside argvFor, not a row a newer or older build // wrote. @@ -351,121 +366,6 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, m) } return argv, nil - case "plans": - if err := plainPlanIDs(a.given); err != nil { - return nil, err - } - if r := str("repository"); r != "" { - argv := []string{"plans", "--what-if", r} - if p := str("paths"); p != "" { - argv = append(argv, "--paths", p) - } - if m := str("modules"); m != "" { - argv = append(argv, "--modules", m) - } - if d := str("module-dirs"); d != "" { - argv = append(argv, "--module-dirs", d) - } - // The plan as data, the document delivery-plan answers (novox/hq ADR 0450 §2, issue 1324). - if str("data") == "true" { - argv = append(argv, "--data") - for _, flag := range []string{"head", "base", "removed"} { - if v := str(flag); v != "" { - argv = append(argv, "--"+flag, v) - } - } - } - return argv, nil - } - for _, act := range []string{"stop", "close", "retry", "go"} { - if id := str(act); id != "" { - argv := []string{"plans", act, id} - if act == "retry" { - return argv, nil - } - // Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without. - if w := str("why"); w != "" { - argv = append(argv, "--why", w) - } - if c := str("cause"); c != "" { - argv = append(argv, "--cause", c) - } - return argv, nil - } - } - if id := str("wait"); id != "" { - // Following a plan (novox/hq issue 229): checked here too, so a call asking what no wait can do is - // refused before a command is started. - if _, _, err := planWaitAsked(a.given["until"], a.given["bound"]); err != nil { - return nil, err - } - argv := []string{"plans", "wait", id} - if u := str("until"); u != "" { - argv = append(argv, "--until", u) - } - if b := str("bound"); b != "" { - argv = append(argv, "--bound", b) - } - return argv, nil - } - if id := str("id"); id != "" { - return []string{"plans", id}, nil - } - argv := []string{"plans"} - if n := str("limit"); n != "" { - argv = append(argv, "-n", n) - } - return argv, nil - // The delivery's owner's verbs (novox/hq ADR 0239). - case "delivery-order": - if err := need("members"); err != nil { - return nil, err - } - members, err := membersFlags(str("members")) - if err != nil { - return nil, err - } - return append([]string{"delivery", "order"}, members...), nil - case "deliver": - if err := need("plan"); err != nil { - return nil, err - } - argv := []string{"delivery", "go", str("plan"), "--by", catalogue.DeliverySeat} - if w := str("why"); w != "" { - argv = append(argv, "--why", w) - } - return argv, nil - case "delivery-stop": - if err := need("plan", "why"); err != nil { - return nil, err - } - argv := []string{"delivery", "stop", str("plan"), "--why", str("why")} - if b := str("by"); b != "" { - argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b) - } - // The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299). - if w := str("without"); w != "" { - argv = append(argv, "--without", w) - } - return argv, nil - case "delivery-backups-again": - if err := need("plan", "why"); err != nil { - return nil, err - } - argv := []string{"delivery", "backups-again", str("plan"), "--why", str("why")} - if b := str("by"); b != "" { - argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b) - } - return argv, nil - case "delivery-walks": - argv := []string{"delivery", "walks"} - if id := str("plan"); id != "" { - return append(argv, "--plan", id), nil - } - if n := str("limit"); n != "" { - argv = append(argv, "-n", n) - } - return argv, nil case "try": // A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered. if id := str("id"); id != "" { @@ -488,18 +388,6 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--store-only") } return argv, nil - case "plan": - if err := need("node"); err != nil { - return nil, err - } - if on("files") { - return []string{"plan", str("node"), "--files"}, nil - } - // What a push would change there (novox/hq ADR 0217); diff with files is passed over, and so refused. - if on("diff") { - return []string{"plan", str("node"), "--diff"}, nil - } - return []string{"plan", str("node"), "--json"}, nil case "assign", "unassign": if err := need("node", "module"); err != nil { return nil, err @@ -523,42 +411,6 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"unpin", str("node"), str("provision")}, nil - case "push": - // A push to a named machine waits for that machine's own answer, so the call's record says - // how it went: applied, refused with the machine's reason, or which resources failed and why - // (novox/hq issue 629). How long is the call's `wait`, else the controller's setting `push-wait`; - // neither given waits for nothing, as before. The caller is answered before the push starts - // (answersFirst) and the machines are released once sent, so the wait holds nobody; it is what - // `calls` then shows. A sweep (--behind) does not wait: the slowest machine would hold back - // what is said of the others. - // A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7). - if err := need("why"); err != nil { - return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err) - } - why := []string{"--why", str("why")} - if c := str("cause"); c != "" { - why = append(why, "--cause", c) - } - if n := str("node"); n != "" { - // behind is not read here: given with a machine, it is refused as passed over — naming - // a machine and asking for every machine behind are two requests, and guessing one - // would push a machine nobody named, or not push one somebody did. - wait, err := pushWait(str("wait")) - if err != nil { - return nil, err - } - argv := []string{"push", n, "--wait", wait.String()} - // A running module's data moving is sent only when said, per module (novox/hq ADR 0217). - if m := str("move"); m != "" { - argv = append(argv, "--move", m) - } - return append(argv, why...), nil - } - // No machine: the whole mesh, whether or not behind said so. The command's answer says it - // first, so a caller who meant one machine reads that it was not one. move is not read: a - // machine whose data would move is held and named, and sent by a push that names it. - on("behind") - return append([]string{"push", "--behind", "--wait", "0"}, why...), nil case "hand-act": if err := need("what", "why", "cause"); err != nil { return nil, err @@ -778,16 +630,6 @@ func (a *verbArguments) commandLine() ([]string, error) { } return argv, nil case "upgrade": - if on("backlog") { - return []string{"upgrade", "backlog"}, nil - } - if on("release-backlog") { - argv := []string{"upgrade", "release-backlog", "--why", str("why")} - if c := str("cause"); c != "" { - argv = append(argv, "--cause", c) - } - return argv, nil - } m, policy := str("module"), str("policy") if m == "" { if policy != "" { @@ -798,9 +640,6 @@ func (a *verbArguments) commandLine() ([]string, error) { argv := []string{"upgrade", m} if policy != "" { argv = append(argv, policy) - if on("together") { - argv = append(argv, "--together") - } if w := str("why"); w != "" { argv = append(argv, "--why", w) } @@ -1319,23 +1158,13 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { // At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0). return statusFrom.answer(ctx) } - if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) { + if !readingVerbs[verb] { // Whatever it did, `status` is composed again once it has. defer statusFrom.nudge() } - if verb == "plans" && len(argv) > 1 && argv[1] == "wait" { - // A wait holds its answer up to its bound (novox/hq issue 229), and is answered in full inside it. - asked := "" - if i := slices.Index(argv, "--bound"); i > 0 && i+1 < len(argv) { - asked = argv[i+1] - } - if _, bound, err := planWaitAsked("", asked); err == nil { - link.AnswersWithin(ctx, bound+planWaitStart) - } - } if verb == "try" && len(argv) > 1 && argv[1] == "--id" { // Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229). - link.AnswersWithin(ctx, tryWait+planWaitStart) + link.AnswersWithin(ctx, tryWait+5*time.Second) } if answer, read := readHere(ctx, argv); read { return answer, nil @@ -1346,13 +1175,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { link.Acknowledge(ctx) } answer, err := runTheVerb(ctx, argv) - // **A plan the verb moved is looked at as it ends** (novox/hq ADR 0406 §5, issue 974): the verb ran as a - // child process, whose saves this process does not hear, so `deliver` after an approval, a stop, a retry - // and a send from a verb would otherwise wait for an unrelated event. Even when it failed: it may have - // kept a plan before it did. - if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) { - askToLook() - } return answer, err } } @@ -1363,16 +1185,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { // verb needs no child process. var runTheVerb = runVerb -// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them. -func actsOnAPlan(args map[string]any) bool { - for _, act := range []string{"stop", "close", "retry"} { - if v, _ := args[act].(string); strings.TrimSpace(v) != "" { - return true - } - } - return false -} - // inProcess are the verbs answered by this process rather than by a command it runs: `tools` from // the records, `calls` from what this process served. var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true, @@ -1386,47 +1198,6 @@ var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true, // issue 1937). "held": true} -// pushFileVar names the file the controller's push settings are written to (module.json, `push-wait`). -const pushFileVar = "MESH_PUSH_FILE" - -// pushWait is how long a push to a named machine waits for its answer (novox/hq issue 629): the call's own -// `wait` when given, else the controller's setting `push-wait`, else nothing. No length is written here: the -// setting's default is the module's, and a setting that cannot be read waits for nothing, as a push did -// before, and is said. -func pushWait(given string) (time.Duration, error) { - if strings.TrimSpace(given) != "" { - d, ok := durationSetting(given) - if !ok { - return 0, fmt.Errorf("push: wait %q is no duration: say it as \"90s\", \"2m\", or \"0\" to wait for nothing", given) - } - return d, nil - } - path := os.Getenv(pushFileVar) - if path == "" { - return 0, nil - } - raw, err := os.ReadFile(path) - if err != nil { - fmt.Fprintf(os.Stderr, "the push settings cannot be read (%v); a push waits for nothing\n", err) - return 0, nil - } - var values map[string]any - if err := json.Unmarshal(raw, &values); err != nil { - fmt.Fprintf(os.Stderr, "the push settings are not JSON (%v); a push waits for nothing\n", err) - return 0, nil - } - v, has := values["push-wait"] - if !has || v == "" { - return 0, nil - } - d, ok := durationSetting(v) - if !ok { - fmt.Fprintf(os.Stderr, "the setting push-wait=%v is no duration; a push waits for nothing\n", v) - return 0, nil - } - return d, nil -} - // answersFirst is a command line whose caller is answered before it runs: a push, by its verb or // through `command`. A push sends the machine holding the bus first when its user list changed, the // broker reloads, and a reload forgets every answer the bus was about to permit — so an answer diff --git a/cmd/mesh-controller/seatverbs_schema_integration_test.go b/cmd/mesh-controller/seatverbs_schema_integration_test.go index c66a5ec9..85bd1c16 100644 --- a/cmd/mesh-controller/seatverbs_schema_integration_test.go +++ b/cmd/mesh-controller/seatverbs_schema_integration_test.go @@ -38,7 +38,7 @@ func servedSchemas(t *testing.T) map[string]catalogue.Verb { func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) { for name, v := range servedSchemas(t) { - if inProcess[name] || removedVerbs[name] { + if inProcess[name] || removedVerb(name) { continue } names, switches := declaredArguments(v) @@ -79,7 +79,7 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) { args := sampleArguments(v) args[stranger] = "x" _, err := argvFor(name, args) - if inProcess[name] || removedVerbs[name] { + if inProcess[name] || removedVerb(name) { _, err = readArguments(name, args) } if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) { @@ -100,13 +100,13 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) { func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) { flags := commandFlags(t) - if len(flags["push"]) == 0 || len(flags["plan"]) == 0 { - t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags) + if len(flags["settings"]) == 0 || len(flags["upgrade"]) == 0 { + t.Fatalf("reading the commands' flags found nothing for settings or upgrade: %v", flags) } reached := map[string]map[string]bool{} // verb → flag sets its command lines reach served := servedSchemas(t) for name, v := range served { - if inProcess[name] || removedVerbs[name] || name == "command" { + if inProcess[name] || removedVerb(name) || name == "command" { continue } names, switches := declaredArguments(v) diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index c34ff0f5..06532054 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -56,28 +56,6 @@ var saidOutright = map[string]string{ // what it requires and one argument more; and `node` in particular, for every verb whose schema // does not take a machine. -// The push that was the cause: a machine named is that machine; none named is the whole mesh, and -// naming one beside behind is refused rather than one of the two guessed. -func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) { - argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"}) - if err != nil || strings.Join(argv, " ") != "push g1 --wait 0s --why w" { - t.Fatalf("a named push: %v %v", argv, err) - } - for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} { - argv, err := argvFor("push", args) - if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" { - t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err) - } - } - if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil || - !strings.Contains(err.Error(), `"behind"`) { - t.Fatalf("a named push with behind was taken: %v", err) - } - if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) { - t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err) - } -} - // commandFlags is every flag set this package's commands parse, by the name the set is made with, // and the flags defined on it — read from the source, so a flag added to a command is seen here // without anyone remembering to. @@ -170,19 +148,10 @@ func stringLit(e ast.Expr) (string, bool) { var accountedFlags = map[string]map[string]string{ "status": {"json": "set by the verb: the answer is data"}, "seats": {"json": "set by the verb: the answer is data"}, - "plan": {"json": "set by the verb unless files is asked"}, - "push": { - "wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply", - "all": "withheld: the verb naming no machine is --behind, and a push of every machine is said at a " + - "shell (novox/hq ADR 0217); `command` reaches it", - }, "builds": {"n": "=limit"}, - "plans": {"n": "=limit", "what-if": "=repository"}, // The machine's tunnel key, named as the verb's other arguments are (novox/hq ADR 0169). - "durations": { - "json": "set by the verb: the answer is data", - "all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it", - }, + // `upgrade` takes --together, which only a roll-out of one machine first once meant; no caller reaches it. + "upgrade": {"together": "withheld: nothing rolls out one machine first any more (ADR 0539 §5); `command` reaches it"}, // The desk path of `secret accept` (novox/hq ADR 0259 §10): a value is never an argument of a call. "hand-acts": {"json": "set by the verb: the answer is data"}, "conditions": {"json": "set by the verb: the answer is data"}, diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index 37193269..5ca9659b 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -1,13 +1,15 @@ package main import ( - "context" "errors" "fmt" + "io/fs" "os" "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // `builds` given a build's id no longer reads a log from the bus: the build seat is retired (ADR 0525 clause 7). @@ -99,62 +101,6 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) { } } -// Acts do not block the call. A build is sent, not waited for: the asker reads the build's log by its id. -// A push answers its caller before it starts (answersFirst), so a named push waits for the machine's own -// answer without holding anybody, and the call's record says how it went (novox/hq issue 629). A -// repository given as a forge path is said to be one (issue 176). -func TestActsDoNotBlockTheCall(t *testing.T) { - argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"}) - if strings.Join(argv, " ") != "push one --wait 0s --why w" || !answersFirst(argv) { - t.Fatalf("a named push with no setting waits, or holds its caller: %v", argv) - } -} - -// A11a (ADR 0501 §5; issue 1851): the build, rebuild and replay --register verbs reach a command that refuses them by -// name, naming the replacement, whatever a table of before asked of them; the command lines refuse the same. -func TestTheRemovedVerbsAreRefusedByName(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - said string - }{ - {"build", map[string]any{"repository": "novox/x", "path": "modules/x"}, "`build` is removed"}, - {"build", map[string]any{}, "`build` is removed"}, - {"rebuild", map[string]any{"what": "gitea"}, "`rebuild` is removed"}, - {"replay", map[string]any{"id": "build-1"}, "`replay` is removed"}, - {"queue", map[string]any{}, "`queue` is removed"}, - {"cancel", map[string]any{"id": "build-1"}, "`cancel` is removed"}, - {"clear", map[string]any{"dead": "true"}, "`clear` is removed"}, - {"kill", map[string]any{"id": "build-1"}, "`kill` is removed"}, - {"pause", map[string]any{"node": "ace"}, "`pause` is removed"}, - {"resume", map[string]any{}, "`resume` is removed"}, - // 1318: delivery-plan and forget-checks go too. - {"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x"}, "`delivery-plan` is removed"}, - {"forget-checks", map[string]any{"owner": "novox", "repo": "a", "number": "7"}, "`forget-checks` is removed"}, - } { - argv, err := argvFor(c.verb, c.args) - if err == nil || !strings.Contains(err.Error(), c.said) { - t.Errorf("%s %v: %v %v, want refused saying %q", c.verb, c.args, argv, err, c.said) - } - } - for _, removed := range []string{"build", "rebuild", "forget-checks", "delivery-plan", "queue", "replay"} { - err := removedByName(removed) - if err == nil || !strings.Contains(err.Error(), "`"+removed+"` is removed") || !strings.Contains(err.Error(), "ADR 05") { - t.Errorf("%s: %v", removed, err) - } - } - if err := deliveryCommand(context.Background(), []string{"plan", "--repository", "novox/a"}); err == nil || - !strings.Contains(err.Error(), "`delivery-plan` is removed") { - t.Errorf("delivery plan: %v", err) - } -} - -// What `tools` answers is the seats' records, with each verb's schema. - -// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it -// printed beside the document does not take the document away. The test binary stands in for the -// controller: `-test.run` with a name that matches nothing prints `ok` and a warning about no tests. - // `command` is the generic verb: the command line as given, split as a shell would, nothing added // (novox/hq ADR 0154, ADR 0175). It once carried an operator's `node account g14 jochen` too; a node's // accounts are the controller's terminal's alone since ADR 0266 (TestNoVerbSetsANodesAccounts). @@ -286,39 +232,57 @@ func TestAQuotedValueRoundTrips(t *testing.T) { } } -// A named push waits as long as the call says, else as long as the controller's setting push-wait says, else -// not at all: no length is the code's own (novox/hq issue 629). -func TestAPushWaitsAsTheCallOrTheSettingSays(t *testing.T) { - file := filepath.Join(t.TempDir(), "push.json") - if err := os.WriteFile(file, []byte(`{"push-wait": "2m"}`), 0o600); err != nil { - t.Fatal(err) +// **A verb release 6 removed is refused by name, with what replaces it** (ADR 0539, ADR 0429 §8, issue 1945 B10): every +// verb of catalogue.RemovedControllerVerbs is answered "no longer exists" naming itself and its replacement, whatever +// arguments it is given, and the tool-call path and the terminal's command line give the same words. +func TestTheVerbsReleaseSixRemovedAreRefusedByName(t *testing.T) { + if len(catalogue.RemovedControllerVerbs) == 0 { + t.Fatal("the SDK lists no removed controller verb") } - t.Setenv(pushFileVar, file) - argv, err := argvFor("push", map[string]any{"node": "one", "why": "w"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 2m0s --why w" { - t.Fatalf("the setting was not the default: %v %v", argv, err) + for verb, why := range catalogue.RemovedControllerVerbs { + argv, err := argvFor(verb, map[string]any{"node": "anchor", "why": "w"}) + if err == nil || len(argv) != 0 { + t.Errorf("%s was composed into %v", verb, argv) + continue + } + for _, want := range []string{"`" + verb + "`", "no longer exists", why} { + if !strings.Contains(err.Error(), want) { + t.Errorf("%s: the refusal %q lacks %q", verb, err, want) + } + } + if cmd := removedByName(verb); cmd == nil || cmd.Error() != err.Error() { + t.Errorf("%s: the terminal says %v, the tool says %v", verb, cmd, err) + } } - argv, err = argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "30s"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 30s --why w" { - t.Fatalf("the call's wait did not win over the setting: %v %v", argv, err) - } - argv, err = argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "0"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 0s --why w" { - t.Fatalf("a wait of 0 did not wait for nothing: %v %v", argv, err) - } - if _, err := argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "soon"}); err == nil || - !strings.Contains(err.Error(), "no duration") { - t.Fatalf("a wait that is no duration was taken: %v", err) - } - if err := os.WriteFile(file, []byte(`{"push-wait": "later"}`), 0o600); err != nil { +} + +// The controller's size, printed against the 111,386 non-test lines its repository held before release 6 (acceptance test 20, ADR +// 0539 §17): raw lines of every non-test Go file of the module, which the pull request reports. +func TestTheControllersNonTestLinesArePrinted(t *testing.T) { + const before = 111386 + total := 0 + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && (d.Name() == ".git" || d.Name() == "testdata") { + return filepath.SkipDir + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + total += strings.Count(string(raw), "\n") + return nil + }) + if err != nil { t.Fatal(err) - } - argv, err = argvFor("push", map[string]any{"node": "one", "why": "w"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 0s --why w" { - t.Fatalf("an unreadable setting did not fall back to waiting for nothing: %v %v", argv, err) } - argv, _ = argvFor("push", map[string]any{"why": "w"}) - if strings.Join(argv, " ") != "push --behind --wait 0 --why w" { - t.Fatalf("a sweep waits: %v", argv) + t.Logf("non-test Go lines: %d against %d before release 6 (%+d)", total, before, total-before) + if total >= before { + t.Errorf("the controller holds %d non-test lines, no fewer than the %d it held before release 6", total, before) } } diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index f2d5aae9..2b6dd79c 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -651,13 +651,14 @@ func rotateKept(ctx context.Context, node, module, name string, why handActFlags machines = []string{node} } if len(machines) == 1 { - fmt.Printf("sending %s, so %s starts again on the new value:\n", node, module) + fmt.Printf("composing %s, so %s starts again on the new value once its want names it:\n", node, module) } else { - fmt.Printf("shared with every consumer; sending %s together:\n", strings.Join(machines, ", ")) + fmt.Printf("shared with every consumer; composing %s together:\n", strings.Join(machines, ", ")) } - if err := sendTo(ctx, open, machines); err != nil { - return fmt.Errorf("%w\n\nThe new value is sealed and not yet delivered; what runs keeps the old "+ - "one until the machines next apply. Fix the cause and run `push --behind`", err) + // Composed (novox/hq ADR 0539 clause 5): a new revision of the compositions that use it. + if _, err := composeModules(ctx, open, machines); err != nil { + return fmt.Errorf("%w\n\nThe new value is sealed and its compositions are not all written; what runs "+ + "keeps the old one. Fix the cause and move it again", err) } return nil } diff --git a/cmd/mesh-controller/send_key_refused.go b/cmd/mesh-controller/send_key_refused.go deleted file mode 100644 index 5c7c9287..00000000 --- a/cmd/mesh-controller/send_key_refused.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "sync" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and -// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes -// the key again; the look that writes it leaves it out of the set, and the condition clears. -const ( - kindSendKeyRefused = "send-key-refused" - sendKeyRefusedID = "DL" -) - -var refusedSendKeys = struct { - mu sync.Mutex - by map[silentWait]error -}{} - -// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found. -func noteRefusedSendKeys(refused map[silentWait]error) { - refusedSendKeys.mu.Lock() - defer refusedSendKeys.mu.Unlock() - refusedSendKeys.by = refused -} - -// observeRefusedSendKeys is one condition per plan and machine whose send key was refused. -func observeRefusedSendKeys() []conditions.Observation { - refusedSendKeys.mu.Lock() - waits := make([]silentWait, 0, len(refusedSendKeys.by)) - why := map[silentWait]error{} - for w, err := range refusedSendKeys.by { - waits = append(waits, w) - why[w] = err - } - refusedSendKeys.mu.Unlock() - sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) }) - seen := map[[2]string]bool{} - var out []conditions.Observation - for _, w := range waits { - if seen[[2]string{w.Plan, w.Node}] { - continue // a plan sends a machine one tier at a time; one condition says it - } - seen[[2]string{w.Plan, w.Node}] = true - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node, - Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning, - Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+ - "names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node), - Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])}) - } - return out -} - -// probeSendKeyRefused reads what the newest look of the plans could not write. -func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) { - return observeRefusedSendKeys(), nil -} diff --git a/cmd/mesh-controller/sendable_integration_test.go b/cmd/mesh-controller/sendable_integration_test.go index ec51664d..da974574 100644 --- a/cmd/mesh-controller/sendable_integration_test.go +++ b/cmd/mesh-controller/sendable_integration_test.go @@ -34,8 +34,8 @@ func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { t.Fatal(err) } if string(body) != convergedBefore { - t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s", - body, convergedBefore) + // One line, got first: the verdict keeps only a failure's first line (issue 1921), and the bytes are what to capture. + t.Fatalf("a converged declaration changed; an older host parses this strictly: got %s want %s", body, convergedBefore) } if bytes.Contains(body, []byte(`"adoption"`)) { t.Fatal("a converged declaration names adoption; an older host would refuse it") @@ -80,25 +80,6 @@ func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { t.Fatalf("taken is not a list on the wire, even empty: %s", body) } - // The digest the mesh compares is the digest of what is sent: status and push agree. - would, err := wouldSend(ctx, open, mustNodes(t, open)) - if err != nil { - t.Fatal(err) - } - if would["anchor"] != digestOf(body) { - t.Fatal("the digest the mesh compares is not of the declaration push sends") - } - - // plan --json prints that same envelope. - printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) }) - var compact bytes.Buffer - if err := json.Compact(&compact, []byte(printed)); err != nil { - t.Fatalf("plan --json is not JSON: %v\n%s", err, printed) - } - if digestOf(compact.Bytes()) != digestOf(body) { - t.Fatalf("plan --json shows something other than what push sends:\n%s", printed) - } - // Taking the module moves its resources out of untaken. if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { t.Fatal(err) diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index 0616525f..5c32546e 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -47,7 +47,11 @@ func composed(t *testing.T, open *stores, node string) sendable { // machine's own resolver knows the mesh's names now — and left this string carrying it, so the // guard failed for a day and nothing ran it. A field an older host never sees is the one change // this guard permits; a field it would refuse is the one it exists to catch. -const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}` +// +// Re-captured for release 6 (issue 1945): every composition is for a node-engine of release 6, which reads a module per +// resource (readNode sets ReadsModules), so each resource carries its "module". An older host that never said it reads them is +// no longer sent a declaration; the field is the wire's own since issue 1145. +const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","module":"hello-web","path":"/var/lib/hello-web/index.html","type":"file"},{"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","module":"hello-web","name":"hello-web","type":"container"},{"id":"hello-web.served","module":"hello-web","path":"/var/lib/hello-web","type":"directory"}]}` func mustNodes(t *testing.T, open *stores) []inventory.Node { t.Helper() diff --git a/cmd/mesh-controller/sent_required_by_integration_test.go b/cmd/mesh-controller/sent_required_by_integration_test.go deleted file mode 100644 index 5bc15604..00000000 --- a/cmd/mesh-controller/sent_required_by_integration_test.go +++ /dev/null @@ -1,101 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - client := aClientModule() - moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) - moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} - kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} - desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} - for _, m := range []catalogue.Manifest{client, moved, kept, desk} { - register(t, open, m) - } - for _, m := range []string{client.Module, desk.Module} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, - SaidAt: time.Now()}); err != nil { - t.Fatal(err) - } - - plan, settings, err := resolveNode(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { - t.Fatalf("the bots are on laptop because %v", plan.Because) - } - declared, err := declarationFor(ctx, open, "laptop", plan, settings) - if err != nil { - t.Fatal(err) - } - var movedToken, keptToken, movedConf map[string]any - for _, r := range declared.Resources { - switch r["id"] { - case "moved-bot.needs-token": - movedToken = r - case "kept-bot.needs-token": - keptToken = r - case "moved-bot.conf": - movedConf = r - } - } - if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) - } - if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) - } - if keptToken == nil || keptToken["sealed"] == nil { - t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) - } - if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { - t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) - } - if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { - t.Fatalf("the send hands %v", declared.plans) - } - - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", - Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { - t.Fatal(err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror := inv.PlanValuesForTest(t) - var value inventory.AssignmentMirrored - if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { - t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) - } - if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || - len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || - len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { - t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) - } - if _, has := mirror["laptop.kept-bot"]; has { - t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) - } -} diff --git a/cmd/mesh-controller/sent_required_by_test.go b/cmd/mesh-controller/sent_required_by_test.go deleted file mode 100644 index eb5a1113..00000000 --- a/cmd/mesh-controller/sent_required_by_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the -// assignments mirror gives that plan a value of its own; an assigned module's plan is kept as the bytes it was. -func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) { - plan := catalogue.Plan{Templates: catalogue.PlanTemplates{Source: "http://store:5000/v2/notify/files", - Digest: "sha256:" + strings.Repeat("d", 64)}, - Files: []catalogue.PlanFile{{ID: "rc", Path: "/home/op/rc", Template: "rc", OwnerAtRuntime: "owned"}}} - r := sentRecordOf(t.Context(), sendable{plans: map[string]catalogue.Plan{"dunst": plan, "notify": plan}, - requiredBy: map[string]string{"notify": "desk"}}) - kept := string(r.plans) - want := `{"dunst":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + - strings.Repeat("d", 64) + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}]},` + - `"notify":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + strings.Repeat("d", 64) + - `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}],"required-by":"desk"}}` - if kept != want { - t.Fatalf("the send keeps\n%s\nwant\n%s", kept, want) - } -} - -// **A module only pulled in by another is handed on a machine that can take it** (issue 765): on a contract-11 -// node running a client, a moved module that nothing assigned there, required by one that is, has its files and its -// own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored -// under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before. diff --git a/cmd/mesh-controller/settings_walk.go b/cmd/mesh-controller/settings_walk.go deleted file mode 100644 index 08fd5aa4..00000000 --- a/cmd/mesh-controller/settings_walk.go +++ /dev/null @@ -1,281 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/compose" - "github.com/novox/mesh-controller/internal/inventory" -) - -// An approved settings change is sent by a walk at once, and the operator's one Approve is its push (novox/hq ADR -// 0344, issue 641). -// -// On 2026-10-13 the operator approved a betterbird proposal on Telegram and nothing happened: the warrant set the -// layer, and the push stayed a separate act (ADR 0277 §18) that nobody took. Now the warrant that sets the layer -// opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the -// builds. Its order is every machine running the module that the layer reaches, the control node last; each is -// sent and its module judged there, from the machine's report on that send, before the next is sent. The send is -// the layer alone: every module there stays at the build its machine was last sent (keepingBuildsForSettings), so a -// build held for a person (ADR 0242) or waiting for a gate stays where it is — a policy of `record` holds a build, -// never a setting — and a send that would move a running module's data stops the walk (ADR 0217 §3). The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's -// channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing -// else. - -// settingsWalkPrefix names a settings walk's id. -const settingsWalkPrefix = "settings-" - -// settingsWalkFor is what a settings walk says it is for, where a merge's walk says its repository. -func settingsWalkFor(s inventory.PlanSettings) string { - where := s.Node - if where == "" { - where = "the whole mesh" - } - return fmt.Sprintf("the settings of %s on %s", s.Module, where) -} - -// settingsWalkOrder is the machines a layer reaches, in the order a settings walk sends them: every machine -// running the module, or the one the layer names when it runs it there, sorted, the control node last. Pure. -func settingsWalkOrder(node string, running, controllers []string) []string { - var order, last []string - for _, n := range running { - if node != "" && n != node { - continue - } - if slices.Contains(controllers, n) { - last = append(last, n) - } else { - order = append(order, n) - } - } - sort.Strings(order) - sort.Strings(last) - return append(order, last...) -} - -// openSettingsWalk opens the walk that sends an approved layer, and answers the words of what it does: the -// machines it sends, or that no machine runs the module and nothing is sent. Called on the operator's warrant, -// after the layer is set. -var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanSettings) (string, error) { - inv := open.inventory - running, err := inv.Running(ctx, s.Module) - if err != nil { - return "", err - } - controllers, err := inv.Running(ctx, catalogue.ControllerSeatName) - if err != nil { - return "", err - } - order := settingsWalkOrder(s.Node, running, controllers) - if len(order) == 0 { - return fmt.Sprintf("no machine runs %s there, so there is nothing to send", s.Module), nil - } - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return "", err - } - defer release() - now := time.Now().UTC() - p := inventory.Plan{ID: fmt.Sprintf("%s%d", settingsWalkPrefix, now.UnixNano()), Repository: settingsWalkFor(s), - Created: now, State: inventory.PlanRolling, Tiers: [][]string{{s.Module}}, Modules: map[string]*inventory.PlanModule{}, - Release: &inventory.PlanRelease{Order: order, Settings: &s}, - Note: fmt.Sprintf("approved (ask %s): sent to %s, one machine at a time, each judged", s.Ask, - strings.Join(order, ", "))} - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - fmt.Printf("%s: %s — %s\n", p.ID, p.Repository, p.Note) - return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil -} - -// settingsDataMoves is what a settings walk's send to a machine would move of a running module's data: composed as -// that send composes it, every build kept, and compared with what the machine was last sent (ADR 0217 §3). -var settingsDataMoves = func(ctx context.Context, open *stores, node string) ([]dataMove, error) { - keeping := keepingBuildsForSettings(keepingRecorded(ctx), node) - keeping = withInputCache(keeping) - if _, err := generatorsOf(keeping, open); err != nil { - return nil, err - } - if _, _, err := resolveNode(keeping, open, node); err != nil { - return nil, err - } - declared, _, err := composeQuestion(keeping, open, node, compose.Reading, false) - if err != nil { - return nil, err - } - body, err := declared.Body() - if err != nil { - return nil, err - } - return heldMoves(keeping, open.inventory, node, body, nil) -} - -// judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A -// settings walk sends a machine only when no other walk judges there: its send would set aside the declaration -// that walk's gate reads the machine's report against. -var judgingOn = func(ctx context.Context, open *stores, self, node string) (string, error) { - plans, err := open.inventory.OpenPlans(ctx) - if err != nil { - return "", err - } - return judgingIn(plans, self, node), nil -} - -// judgingIn is judgingOn over plans read. Pure. -func judgingIn(plans []inventory.Plan, self, node string) string { - judging := func(g *inventory.PlanGate) bool { - return g != nil && g.Verdict == "" && slices.Contains(g.Machines, node) - } - for _, p := range plans { - if p.ID == self || !p.Open() { - continue - } - if p.Release != nil { - if judging(p.Release.Gate) { - return p.ID - } - continue - } - for _, s := range p.Modules { - if s != nil && judging(s.Gate) { - return p.ID - } - } - } - return "" -} - -// advanceSettingsWalk takes one step of a settings walk: the next machine sent and its module judged there, or the -// machine being judged judged once more; done when every machine is. -func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { - r, s := p.Release, p.Release.Settings - now := time.Now().UTC() - wait := func(note string) (bool, error) { - changed := p.Note != note - p.Note = note - return changed, nil - } - if r.Gate == nil { - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, err - } - for r.Next < len(r.Order) { - node := r.Order[r.Next] - if !heard[node] { - r.Skipped = append(r.Skipped, node) - r.Next++ - continue - } - if id, err := judgingOn(ctx, open, p.ID, node); err != nil { - return false, err - } else if id != "" { - return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id)) - } - // **A running module's data that would move is a person's word** (ADR 0217 §3): the walk stops before - // that machine, failed and said, and `push --move` sends it. - held, err := settingsDataMoves(ctx, open, node) - if errors.Is(err, errSettingsCannotKeep) { - return failBefore(p, node, err.Error()) - } - if err != nil { - return false, fmt.Errorf("whether %s's data would move could not be read: %w", node, err) - } - if len(held) > 0 { - var said strings.Builder - sayDataHeld(&said, node, held) - return failBefore(p, node, "a running module's data would move there — "+ - strings.Join(strings.Fields(said.String()), " ")) - } - // A build that cannot be kept is refused by the composition above, which is the send's own, and fails the - // walk there: the send flattens its refusals into words, so it is not looked for again here. - sent, grants, err := sendJudged(keepingBuildsForSettings(ctx, node), open, node) - if err != nil { - return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err) - } - // **The module's build is the one the gate judges** (issue 641 review): kept by this send, it is what a - // crossing send — a merge's walk sending the machine a new build of the module — supersedes, so this gate - // never reads that send's report as its own. - kept := "" - if builds, known, err := open.inventory.SentBuilds(ctx, node); err == nil && known { - kept = builds[s.Module] - } - r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now, - From: kept, To: kept, Sent: sentNow(ctx, open.inventory, sent), Grants: grants} - p.Note = fmt.Sprintf("sent %s %s (ask %s), every build there kept; judging %s there", node, - settingsWalkFor(*s), s.Ask, s.Module) - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - p.State, p.Tier = inventory.PlanDone, len(p.Tiers) - p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, strings.Join(r.Done, ", ")) - if len(r.Done) == 0 { - // Sent nowhere is not delivered: said as the walk's failure, so the operator is not told it landed. - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s (ask %s) was sent to no machine", settingsWalkFor(*s), s.Ask) - } - if len(r.Skipped) > 0 { - p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ") - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - g := r.Gate - node := r.Order[r.Next] - verdict, err := judgeMoves(ctx, open, g, []judged{{module: s.Module, node: node}}, now) - if err != nil { - return false, err - } - switch verdict { - case "": - return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g))) - case inventory.GatePassed: - r.Done = append(r.Done, node) - r.Next++ - r.Gate = nil - return true, nil - case inventory.GateSuperseded: - p.State = inventory.PlanSuperseded - p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; the layer is set, and the "+ - "machines not yet sent take it at their next send", node, g.Why) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+ - "replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + unreached(r) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// failBefore ends a settings walk failed before it sent a machine, saying why, and that the layer stays. -func failBefore(p *inventory.Plan, node, why string) (bool, error) { - r, s := p.Release, p.Release.Settings - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s not sent to %s: %s; the layer stays as the operator approved it", settingsWalkFor(*s), - node, why) + unreached(r) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// unreached is what a stopped settings walk left without the layer, in words: the machines after the one it stopped -// at, and those passed over as away (ADR 0344 §6). Empty when there are none. -func unreached(r *inventory.PlanRelease) string { - var out string - if r.Next+1 < len(r.Order) { - out += "; not sent: " + strings.Join(r.Order[r.Next+1:], ", ") - } - if len(r.Skipped) > 0 { - out += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - return out -} diff --git a/cmd/mesh-controller/settings_walk_integration_test.go b/cmd/mesh-controller/settings_walk_integration_test.go deleted file mode 100644 index 5f11ec27..00000000 --- a/cmd/mesh-controller/settings_walk_integration_test.go +++ /dev/null @@ -1,439 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", - "content": "x = ${setting:x}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - // A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7). - if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil { - t.Fatal(err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans)) - } - set := setLayerIn(open) - now := time.Now() - values := map[string]any{"x": "1"} - p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { - t.Errorf("the warrant's words: %q", words) - } - plans, err := open.inventory.OpenPlans(ctx) - if err != nil || len(plans) != 1 { - t.Fatalf("open walks: %+v %v", plans, err) - } - w := plans[0] - if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || - !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || - !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { - t.Fatalf("the settings walk: %+v %+v", w, w.Release) - } - - // A machine's layer where the module does not run: set, and nothing to send. - q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") - if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { - t.Fatalf("a layer nobody runs: %q %v", words, err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { - t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) - } - - // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. - was := openSettingsWalk - openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { - return "", fmt.Errorf("the plans are not to be had") - } - t.Cleanup(func() { openSettingsWalk = was }) - again := map[string]any{"x": "2"} - byHand := map[string]any{"x": "0"} - r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true, - From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)} - if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ - "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { - t.Errorf("an unopened walk: %v", err) - } -} - -func aSettledMesh(t *testing.T) *walkMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{}, - heard: map[string]bool{"anchor": true, "laptop": true}} - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - old := time.Now().Add(-2 * time.Hour) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { - t.Fatal(err) - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil } - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil } - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - w.sent = append(w.sent, append([]string(nil), names...)) - w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0])) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if w.broken[r.Node] { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - return f, nil - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - settingsDataMoves = wasData - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - return w -} - -func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) - if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { - t.Fatalf("%q %v", words, err) - } - advancePlans(ctx, w.open) - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) - } - p := w.walk(t) - if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { - t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) - } - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", w.sent) - } - p = w.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || - p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { - t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) - } - // It said its start and its end, so the operator is told once (ADR 0308). - if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { - t.Fatalf("the walk's start and end are not marked: %+v", p.Times) - } - // Nothing waits, and no walk of waiting builds opened beside it. - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { - t.Fatalf("%+v %v", r, err) - } -} - -func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - w.broken["anchor"] = true - if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil { - t.Fatal(err) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || - !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) - } - if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" { - t.Fatalf("the layer after the failure: %v %v %v", layer, has, err) - } - // A build that waits for a gate is still walked: the failed settings walk holds nothing back. - w.broken["anchor"] = false - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) - } -} - -func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - gateEvery = 0 - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone { - t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note) - } - // What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor") - if err != nil || kept["app"] != "c1" { - t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err) - } - if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { - t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) - } - // And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see. - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - waiting, _ := machineMoves(ctx, w.open, f, "anchor", false) - if len(waiting) != 1 || waiting[0].Module != "app" { - t.Fatalf("anchor waits for %+v, want app's c2", waiting) - } - if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 { - t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err) - } -} - -func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - gateEvery = 0 - w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}} - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) || - !strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") || - !strings.Contains(p.Note, "push laptop --move app") { - t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note) - } - - v := aSettledMesh(t) - v.heard = map[string]bool{"laptop": true} - gateEvery = 0 - if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, v.open) - } - p = v.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) || - !strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") { - t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note) - } - - u := aSettledMesh(t) - u.heard = map[string]bool{} - if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, u.open) - if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 { - t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) - } -} - -func TestACrossingSendSupersedesASettingsGate(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" { - t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate) - } - // Another walk sends anchor app's c2 while the settings gate judges there. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor", - Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, w.open) - p = w.walk(t) - if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded || - !strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 { - t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent) - } -} - -func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - // What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on. - if _, err := inv.AddNode(ctx, "fresh"); err != nil { - t.Fatal(err) - } - if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) { - t.Fatalf("a settings send to a machine never sent: %v", err) - } - w.heard = map[string]bool{"laptop": true} - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { - return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if p.State != inventory.PlanFailed || len(w.sent) != 0 || - !strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") || - !strings.Contains(p.Note, "not heard from, left as they were: anchor") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - // It is not open, so a build waiting for a gate is walked. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err) - } -} diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go deleted file mode 100644 index 2d9e0c18..00000000 --- a/cmd/mesh-controller/settings_walk_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "context" - "reflect" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take -// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a -// time, judging the module on each; the ask says so before the Approve, and nothing names a push. - -// The order: every machine running the module, or the one the layer names, the control node last. -func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) { - running := []string{"shanks", "novox", "g14", "ace"} - for _, c := range []struct { - node string - want []string - }{ - {"", []string{"ace", "g14", "shanks", "novox"}}, - {"g14", []string{"g14"}}, - {"novox", []string{"novox"}}, - {"elsewhere", nil}, - } { - if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) { - t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want) - } - } -} - -// A settings walk sends a machine only when no other walk judges there. -func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) { - judging := &inventory.PlanGate{Machines: []string{"g14"}} - passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed} - plans := []inventory.Plan{ - {ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}}, - {ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}}, - {ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, - {ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, - } - if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" { - t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id) - } - if id := judgingIn(plans[1:2], "x", "g14"); id != "" { - t.Errorf("a passed gate judges: %q", id) - } - if id := judgingIn(plans, "x", "shanks"); id != "" { - t.Errorf("on shanks %q judges", id) - } -} - -// The ask says, before the Approve, that the layer is sent at once and judged, and that its end is told as every -// walk's is; neither the ask nor the proposer's answer names a push any more. -func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { - for _, c := range []struct { - in proposeInput - where string - had bool - }{ - {proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false}, - {proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true}, - {proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true}, - } { - r := newProposerRig(t) - if c.had { - r.before, r.had = map[string]any{"x": "0"}, true - } - words, err := r.pr.propose(context.Background(), c.in) - if err != nil { - t.Fatal(err) - } - q := r.askSent(t) - if err := q.Check(r.now); err != nil { - t.Fatalf("the ask is refused: %v", err) - } - // At a glance (ADR 0345): the summary says it is sent at once by a walk, one machine at a time; the rest of - // what Approve does is the Details', which the router notes after the answers. - if !strings.Contains(q.Explanation, "sent at once by a walk to "+c.where+", one machine at a time, each judged") || - !strings.Contains(q.Details, "and sent at once to "+c.where+", one machine at a time, each judged, "+ - "every build there kept; its end is told as every walk's is") { - t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation) - } - approve := q.Options[0] - if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") { - t.Errorf("Approve does %q", approve.Does) - } - if !strings.Contains(words, "a walk sends it to "+c.where+" at once") { - t.Errorf("the proposer's answer:\n%s", words) - } - for _, said := range []string{q.Explanation, q.Whole, approve.Does, words} { - if strings.Contains(said, "push") { - t.Errorf("a push is still named, as if a second act were needed:\n%s", said) - } - } - } -} - -// ---- on the real stores ----------------------------------------------------------------------------- - -// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a -// machine that does not run the module opens none and says so. - -// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at -// once, recorded, and judged from the reports, with a machine named broken failing what it was sent. -type walkMesh struct { - open *stores - sent [][]string - broken map[string]bool - // kept says, per send, whether it kept every build of its machine; moves is the data a machine's send would move. - kept []bool - moves map[string][]dataMove - heard map[string]bool -} - -func (w *walkMesh) walk(t *testing.T) inventory.Plan { - t.Helper() - plans, err := w.open.inventory.RecentPlans(t.Context(), 10) - if err != nil { - t.Fatal(err) - } - for _, p := range plans { - if p.Release != nil && p.Release.Settings != nil { - return p - } - } - t.Fatal("no settings walk") - return inventory.Plan{} -} - -// The walk sends the first machine alone and judges the module there before it sends the next, and ends -// delivered on both; nothing waited for a gate, so each send carried the layer alone. - -// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the -// failure holds back no walk of waiting builds. - -// Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or -// waiting for a gate stays where it is (ADR 0344 §3). - -// A machine whose send would move a running module's data stops the walk there, failed and said with the push that -// sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached -// no machine is not said delivered. - -// The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new -// build of the module while the settings gate judges there — supersedes it, and its report is never read as the -// settings send's (issue 641 review, finding 1). - -// A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is -// gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a -// failure names the machines passed over as away (issue 641 review, findings 2 and 3). diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 7ee34b89..18bfea30 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -58,8 +58,6 @@ const ( controlNodeUrgentAfter = 30 * time.Minute // reportAtLeast is the least a machine is given to report a send (S2). reportAtLeast = 2 * time.Minute - // tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at. - tierAtLeast = planWaitBound // loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4). loopDeafAfter = 2 * time.Minute // callDefault is the bound of a verb that declares none (S7); push's and build's are longer. @@ -102,20 +100,6 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard }) }}, - {Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent", - Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep", - Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1, - needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports, - newest: func(f *signalFacts) time.Time { - return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt }) - }}, - {Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered", - Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional", - Kind: "stalled", Severity: conditions.Warning, Phase: 1, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans, - newest: func(f *signalFacts) time.Time { - return newestOf(f.plans, func(p planFacts) time.Time { return p.entered }) - }}, {Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller", Trigger: "while its consumers have pending messages", Bound: "2 min", Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1, @@ -189,58 +173,6 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.handActs, func(a link.HandAct) time.Time { return a.At }) }}, - {Row: "S16", Signal: "a walk waiting for its delivery's word is let go", Emitter: "mesh-delivery, through deliver", - Trigger: "each merge whose walk waits (novox/hq ADR 0239)", - Bound: "30 min, then urgent after 4 h — raised by the controller whatever mesh-delivery says of itself, " + - "naming `plans go` as the way on", - Kind: kindWalkWaiting, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchWaits, - newest: func(f *signalFacts) time.Time { - return newestOf(f.waits, func(w waitFacts) time.Time { return w.since }) - }}, - {Row: "S18", Signal: "a batch of merges is cut into its walk", Emitter: "controller's merge window", - Trigger: "each batch (novox/hq ADR 0276)", - Bound: "a minute past merge-window-at-most, while no walk is open: the controller failed to cut it", - Kind: kindBatchNotCut, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesNotCut, - newest: func(f *signalFacts) time.Time { - return newestOf(f.batches, func(b batchFacts) time.Time { return b.atMost }) - }}, - {Row: "S19", Signal: "a batch waiting behind an open walk is cut when it ends", Emitter: "controller's merge window", - Trigger: "each batch closed while a walk is open (novox/hq ADR 0276)", - Bound: "the walk's bound, a tier's bound for each of its tiers, from when the batch closed: naming the walk", - Kind: kindBatchBehindWalk, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesBehind, - newest: func(f *signalFacts) time.Time { - return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed }) - }}, - {Row: "S22", Signal: "the last member of a delivery group merges", Emitter: "mesh-delivery's group, held by the controller's merge window", - Trigger: "each batch held for a group with a member still to merge (novox/hq ADR 0418)", - Bound: "merge-window-at-most from the group's first merge: naming the member and why it did not merge; " + - "nothing of the group is delivered while it waits", - Kind: kindGroupNotMerged, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchGroupsNotMerged, - newest: func(f *signalFacts) time.Time { - return newestOf(f.batches, func(b batchFacts) time.Time { return b.created }) - }}, - {Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan", - Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)", - Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " + - "words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " + - "from the plan's last change, and ended when the plan is no longer failed", - Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed, - newest: func(f *signalFacts) time.Time { - return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since }) - }}, - {Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator", - Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)", - Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " + - "machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " + - "pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.", - Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld, - newest: func(f *signalFacts) time.Time { return time.Time{} }}, {Row: "S25", Signal: "a module left out of a send because the artifact store lacks an image it names", Emitter: "controller's send", Trigger: "each send whose declaration names an image of the store that the store does not hold (novox/hq issue 1742)", Bound: "none: raised on the first look at a registered build that names such an image, naming the module, each " + @@ -249,34 +181,6 @@ var signalsTable = []signalRow{ Kind: kindImageMissing, Severity: conditions.Warning, Phase: 3, needs: func(f *signalFacts) error { return f.imagesErr }, watch: watchMissingImages, newest: func(f *signalFacts) time.Time { return time.Time{} }}, - {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", - Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", - Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + - "machine applied; cleared an hour after the last", - Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2, - needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals, - newest: func(f *signalFacts) time.Time { - return newestOf(f.refusedSends, func(s inventory.Send) time.Time { - if s.RefusedAt == nil { - return time.Time{} - } - return *s.RefusedAt - }) - }}, - {Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan", - Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)", - Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " + - "since when and the mesh-controller.bus call; cleared once the bus's machine has been sent the build the mesh holds", - Kind: kindBusStepWaiting, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { - if f.plansErr != nil { - return f.plansErr - } - return f.busErr - }, watch: watchBusWaits, - newest: func(f *signalFacts) time.Time { - return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) - }}, {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " + "is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " + @@ -370,128 +274,6 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation { return out } -func watchReports(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, m := range f.machines { - if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() { - continue - } - if m.silent(f.now) { - continue // silent: S1 says it, and a silent machine reports nothing - } - bound := max(reportAtLeast, 3*m.lastApply) - waited := f.now.Sub(m.sentAt) - if waited <= bound { - continue - } - out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, - Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning, - Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)", - m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)), - Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))}) - } - return out -} - -func watchPlans(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, p := range f.plans { - // Held only by the bus's planned step (S17, novox/hq issue 336): a wait - // for a person, said as itself, not a walk running late. - if p.bus { - continue - } - in := f.now.Sub(p.entered) - if in <= p.bound { - continue - } - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled", - Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway", - Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+ - "usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))), - Resolved: "Delivery of " + repoName(p.repository) + " moves again", - Severity: conditions.Warning, - Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s", - p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"), - ago(p.bound), p.waiting), - Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)}) - } - return out -} - -// kindPlanFailed is S23's kind: plan..plan-failed. -const kindPlanFailed = "plan-failed" - -// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no -// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to -// the operator). A plan failed longer ago than this is left to `plans`. -const planFailedWithin = 7 * 24 * time.Hour - -// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when -// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2). -// The condition ends when the plan is no longer failed, because it is no longer among the facts. -func watchPlansFailed(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, p := range f.failed { - if f.now.Sub(p.since) > planFailedWithin { - continue - } - headline, explanation, needs, actions, resolved := planFailedWords(p) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed, - Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator, - Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit), - p.failureSaid()), - Said: p.failureSaid() + p.refusalSaid(), - Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved}) - } - return out -} - -// kindWalkWaiting is S16's kind: plan..waiting. -const kindWalkWaiting = "waiting" - -// watchWaits is S16: a walk that has waited for its delivery's word past its bound, said by the controller -// itself — a delivery's owner that is up and never says go (a bug, a delivery stuck in its own table) would -// otherwise leave a merge waiting for ever with nothing open (novox/hq ADR 0239). -func watchWaits(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, w := range f.waits { - in := f.now.Sub(w.since) - if in <= waitBound { - continue - } - severity := conditions.Warning - if in > waitUrgentAfter { - severity = conditions.Urgent - } - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.id, Kind: kindWalkWaiting, - Severity: severity, - Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+ - "deliveries that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository, - short(w.commit), ago(in), w.awaits, mergesWords(w), w.id), - Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits), - Headline: deliveryName(w.modules, w.repository) + " waiting to start", - Explanation: walkWaitingWords(w, in, severity), - Needs: waitingNeeds(severity), - Actions: waitingActions(w.id, severity), - Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"}) - } - return out -} - -// mergesWords is the merges a waiting walk answers (novox/hq ADR 0276): every delivery it carries, not one -// commit that a supersession may already have ended (issue 362). Its own commit for a walk naming none. -func mergesWords(w waitFacts) string { - if len(w.merges) == 0 { - return short(w.commit) - } - var out []string - for _, m := range w.merges { - out = append(out, m.Repository+"@"+short(m.Commit)) - } - return readableList(out) -} - func watchLoop(f *signalFacts) []conditions.Observation { if f.loop.pending == 0 { return nil diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index bbb0f09a..673a0987 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "os" "path/filepath" "regexp" @@ -30,8 +31,6 @@ func calm(now time.Time) *signalFacts { machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second), sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute), lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}}, - plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2, - entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}}, loop: loopFacts{took: now.Add(-time.Second), pending: 1}, selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute}, lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{}, @@ -42,7 +41,7 @@ func calm(now time.Time) *signalFacts { // refusedBy is n refusals of one writer, the last a moment ago. func refusedBy(now time.Time, epoch int64, n int) []link.WriterRefusals { - return []link.WriterRefusals{{Writer: link.WriterEpoch(epoch), Epoch: epoch, Count: n, + return []link.WriterRefusals{{Writer: fmt.Sprintf("the controller of epoch %d", epoch), Epoch: epoch, Count: n, Receivers: []string{"anchor", "laptop"}, Last: now.Add(-time.Second)}} } @@ -55,18 +54,6 @@ var suppressions = map[string]suppression{ inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) }, past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) }, }, - "S2": { - inside: func(f *signalFacts) { - f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false - }, - past: func(f *signalFacts) { - f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false - }, - }, - "S3": { - inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) }, - past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) }, - }, "S4": { inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) }, past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) }, @@ -116,57 +103,11 @@ var suppressions = map[string]suppression{ past: func(f *signalFacts) { f.facts.taken = f.now.Add(-49 * time.Hour) }, }, // A walk waiting for its delivery's word for 30 minutes (novox/hq ADR 0239). - "S16": { - inside: func(f *signalFacts) { - f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", - since: f.now.Add(-29 * time.Minute)}} - }, - past: func(f *signalFacts) { - f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", - since: f.now.Add(-31 * time.Minute)}} - }, - }, // A batch still assembling past its maximum with no walk open (novox/hq ADR 0276): a minute's grace. - "S18": { - inside: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - atMost: f.now.Add(-59 * time.Second), closed: f.now.Add(-59 * time.Second)}} - }, - past: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - atMost: f.now.Add(-61 * time.Second), closed: f.now.Add(-61 * time.Second)}} - }, - }, // A batch held for a group whose last member has not merged by the window's maximum after the first (ADR 0418). - "S22": { - inside: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - holds: map[string]string{"feat/g": "group feat/g has a member still to merge: novox/two (ready)"}, - created: f.now.Add(-9 * time.Minute), atMostWindow: 10 * time.Minute}} - }, - past: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - holds: map[string]string{"feat/g": "group feat/g has a member still to merge: novox/two (ready)"}, - created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}} - }, - }, // A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449). - "S23": { - inside: func(f *signalFacts) { - f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, - since: f.now.Add(-planFailedWithin - time.Second)}} - }, - past: func(f *signalFacts) { - f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, - since: f.now.Add(-time.Second)}} - }, - }, // A machine holding a move of a recorded module for a person is asked at its first look (novox/hq issue 1732). // Inside: nothing waits there. - "S24": { - inside: func(f *signalFacts) { f.held = nil }, - past: func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} }, - }, // A registered build naming an image the artifact store lacks is raised at the first look (novox/hq issue 1742). "S25": { inside: func(f *signalFacts) { f.images = nil }, @@ -175,37 +116,10 @@ var suppressions = map[string]suppression{ }, }, // A batch queued behind an open walk past that walk's bound, named. - "S19": { - inside: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11", - closed: f.now.Add(-59 * time.Minute), behind: "plan-1", walkBound: time.Hour}} - }, - past: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11", - closed: f.now.Add(-61 * time.Minute), behind: "plan-1", walkBound: time.Hour}} - }, - }, // A send refused because it would replace the bus outside its planned step: said at its first refusal, // whatever the bound (novox/hq issue 336). Inside: a new bus build waits, and no send was refused for it. - "S17": { - inside: func(f *signalFacts) { - f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"}, - machines: []string{"anchor"}} - }, - past: func(f *signalFacts) { - f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"}, - machines: []string{"anchor"}, waits: []busWaitFacts{{plan: "plan-1", repository: "novox/app", - commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}} - }, - }, // A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at // once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago. - "S20": { - inside: func(f *signalFacts) { - f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))} - }, - past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, - }, // A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the // failure is older than one apply interval while it is heard. Inside: exactly five minutes old. "S21": { @@ -489,7 +403,7 @@ func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) { f.machines[0].lastHeard = now.Add(-2 * time.Hour) f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)} - if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 { + if got := append(watchHeartbeats(f), watchTools(f)...); len(got) != 0 { t.Fatalf("a sleeping machine raised %+v", got) } f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)} @@ -526,7 +440,7 @@ func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) { for _, c := range open { keys = append(keys, c.Key) } - for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} { + for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S11.failed"} { if !slices.Contains(keys, want) { t.Errorf("%s is not open while the machines cannot be read: %v", want, keys) } @@ -608,21 +522,6 @@ func TestALeaseLostOrMissingIsSaid(t *testing.T) { } } -// **A walk waiting four hours for its delivery's word is urgent** (S16, novox/hq ADR 0239), and says how on. -func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) { - now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - f := calm(now) - f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", - since: now.Add(-4*time.Hour - time.Minute)}} - got := watchWaits(f) - if len(got) != 1 || got[0].Severity != conditions.Urgent || got[0].Key() != "plan.plan-2.waiting" { - t.Fatalf("four hours of waiting raised %+v", got) - } - if !strings.Contains(got[0].Summary, "plans go plan-2") || !strings.Contains(got[0].Summary, "mesh-delivery") { - t.Fatalf("it does not say how on: %q", got[0].Summary) - } -} - // **A drill is a person's deliberate test, never a repair** — the log of 2026-10-07: two drills of ADR // 0240 recorded through `hand-act record --cause drill` before `hand-act drill` existed raised // `mesh.hand-acts.drill.healer-wanted`. A drill through its own verb never counts, the two recorded diff --git a/cmd/mesh-controller/silent_mid_tier_integration_test.go b/cmd/mesh-controller/silent_mid_tier_integration_test.go deleted file mode 100644 index fc7c8577..00000000 --- a/cmd/mesh-controller/silent_mid_tier_integration_test.go +++ /dev/null @@ -1,211 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1160/AC5, live, with the store and a real bus and no ticker: a plan of two tiers whose first tier has a machine that -// never reports, and whose second tier waits on the first (the plan's own order), passes its first tier after the bound, -// the silent key's expiry the only trigger, and sends its second tier to the machine that reported alone (issue 1160). -func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToTheOthers(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - // A quarter of an hour of silence is two seconds here. - wasSilent := applySilentAfter - applySilentAfter = 2 * time.Second - t.Cleanup(func() { applySilentAfter = wasSilent }) - - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - sentHeard := make(chan *nats.Msg, 32) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - wasSaved := inventory.PlanSaved - inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - - // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. - var mu sync.Mutex - var sends [][]string - asksRecorded(t) // each module is taken from a done key as the walk reaches it - // Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look goes on - // through tier 1's send and overwrites the note of the move before the test reads it (see away_integration_test.go). - heldB := holdKeys(t, "b") - digests := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - mu.Lock() - defer mu.Unlock() - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range []string{"a", "b"} { - builds[m] = current[m].Commit - } - for _, node := range names { - digests++ - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - register := func(m, commit string, asked time.Time) { - t.Helper() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - record := func(id, m, commit string, at time.Time) { - t.Helper() - b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - old := time.Now().Add(-2 * time.Hour) - for _, m := range []string{"a", "b"} { - record("build-"+m+"-1", m, "c1", old) - register(m, "c1", old) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { - t.Fatal(err) - } - } - record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) - register("a", "c2", time.Now().Add(-time.Minute)) - - // A machine's report on what it was sent, heard as the controller hears it. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - seq := int64(0) - reports := func(node string) { - t.Helper() - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - t.Fatalf("%s was sent nothing: %v", node, err) - } - seq++ - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": seq}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - plan := func() inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, "plan-1160") - if err != nil { - t.Fatal(err) - } - return p - } - - built := time.Now().UTC() - p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(), - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, - "b": {}, - }} - first, stopFirst := context.WithCancel(ctx) - defer stopFirst() - waitFirst := startRolloutEvents(first, open, js) - // Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338). - t.Cleanup(func() { stopFirst(); waitFirst() }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - eventually(t, "tier 0 sent to both machines", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 1 - }) - reports("anchor") - time.Sleep(time.Second) - if got := plan().Tier; got != 0 { - t.Fatalf("1160/AC5: the plan went on to tier %d before the bound, with laptop not heard from", got) - } - // The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own. - var moved string - eventually(t, "tier 1 passed once laptop's key expired", func() bool { - if q := plan(); q.Tier == 1 { - moved = q.Note - return true - } - return false - }) - if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") { - t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved) - } - // The move to tier 1 is saved in one step and the look that marks b taken from its key is the next (release_plan.go - // advances the tier, plan_keys.go takeFromKey marks it): read b once that look ran, not in the instant the tier moved, - // which a loaded build node shows before it (group check fcb076c5387f). - eventually(t, "tier 1's module taken from its key", func() bool { - b := plan().Modules["b"] - return b != nil && b.Key - }) - // Tier 1's module is taken from its key once that is built, which the walk's look does by itself. - heldB.Store(false) - askToLook() - eventually(t, "tier 1 sent to anchor alone", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor" - }) -} diff --git a/cmd/mesh-controller/silent_mid_tier_test.go b/cmd/mesh-controller/silent_mid_tier_test.go deleted file mode 100644 index f0d64c99..00000000 --- a/cmd/mesh-controller/silent_mid_tier_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package main - -import ( - "slices" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A machine silent past the bound at a tier between the first and the last is named silent and left as it was, as the -// last tier's is (novox/hq ADR 0282 §1, ADR 0360 §4-5, issue 1160): one rule, walkReports, read by the walk's end and by -// the gates between tiers. - -func silentWalk(now time.Time) (inventory.Plan, []inventory.Reported) { - sent := now.Add(-time.Minute) - long := now.Add(-inventory.ApplySilentAfter - time.Minute) - at := now.Add(-10 * time.Second) - p := inventory.Plan{Tier: 0, Tiers: [][]string{{"engine"}, {"app"}}, Modules: map[string]*inventory.PlanModule{ - "engine": {Rest: map[string]inventory.SentDeclaration{ - "anchor": {Digest: "d-anchor", At: sent}, - "g14": {Digest: "d-g14", At: long}, - "fresh": {Digest: "d-fresh", At: sent}}}, - "app": {}}} - reports := []inventory.Reported{{Node: "anchor", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-anchor"}} - return p, reports -} - -func needsOf(running ...string) []gateNeed { - return []gateNeed{{module: "engine", running: running, since: func(string) time.Time { return time.Time{} }}} -} - -// 1160/AC1: at a middle tier the machine silent past the bound is named, not waited for, and the plan goes on with the -// machines that reported; one still inside the bound is waited for. -func TestAMiddleTierGateNamesTheSilentMachineAndGoesOnWithThoseThatReported(t *testing.T) { - now := time.Now().UTC() - p, reports := silentWalk(now) - silent := silentNodes(p, reports, now, nil) - waiting, quiet, lost := settleTierGate(needsOf("anchor", "g14"), reports, silent) - if len(waiting) != 0 || !slices.Equal(quiet, []string{"g14"}) || lost { - t.Fatalf("1160/AC1: waiting %v, silent %v, lost %v; want g14 named silent and nothing waited for", waiting, quiet, lost) - } - waiting, quiet, _ = settleTierGate(needsOf("anchor", "g14", "fresh"), reports, silent) - if len(waiting) != 1 || waiting[0] != "engine on fresh" || !slices.Equal(quiet, []string{"g14"}) { - t.Fatalf("1160/AC1: a machine inside the bound is still waited for: waiting %v, silent %v", waiting, quiet) - } -} - -// 1160/AC2: the walk's end and the gate read the same silent machines. -func TestTheGateAndTheWalksEndShareOneRuleOfSilence(t *testing.T) { - now := time.Now().UTC() - p, reports := silentWalk(now) - _, _, ended := walkReports(p, reports, now, nil) - got := silentNodes(p, reports, now, nil) - if len(got) != len(ended) || !got["g14"] || !slices.Equal(ended, []string{"g14"}) { - t.Fatalf("1160/AC2: the gate reads %v silent, the walk's end %v", got, ended) - } -} - -// 1160/AC3: a later tier is not sent to a machine silent. -func TestALaterTierIsNotSentToASilentMachine(t *testing.T) { - got := withoutSilent([]string{"anchor", "g14"}, map[string]bool{"g14": true}) - if !slices.Equal(got, []string{"anchor"}) { - t.Fatalf("1160/AC3: a later tier goes to %v; want anchor alone", got) - } - in := []string{"anchor", "g14"} - withoutSilent(in, map[string]bool{"g14": true}) - if !slices.Equal(in, []string{"anchor", "g14"}) { - t.Fatalf("1160/AC3: the machines running it were changed: %v", in) - } -} - -// 1160/AC4: every machine the gate waits on silent: the plan is lost, not gone on as if delivered. -func TestAGateWhoseEveryMachineIsSilentIsLost(t *testing.T) { - now := time.Now().UTC() - p, reports := silentWalk(now) - reports = nil - p.Modules["engine"].Rest = map[string]inventory.SentDeclaration{ - "g14": {Digest: "d", At: now.Add(-time.Hour)}, "laptop": {Digest: "d", At: now.Add(-time.Hour)}} - silent := silentNodes(p, reports, now, nil) - _, quiet, lost := settleTierGate(needsOf("g14", "laptop"), reports, silent) - if !lost || !slices.Equal(quiet, []string{"g14", "laptop"}) { - t.Fatalf("1160/AC4: lost %v, silent %v; want both named and the gate lost", lost, quiet) - } -} diff --git a/cmd/mesh-controller/states_announced.go b/cmd/mesh-controller/states_announced.go index f4f05611..d399528e 100644 --- a/cmd/mesh-controller/states_announced.go +++ b/cmd/mesh-controller/states_announced.go @@ -191,21 +191,12 @@ func composeAnnouncedReads(ctx context.Context, open *stores) { if holder == "" || !behind { return } - held, err := heldMachines(ctx, open, []string{holder}) - if err != nil { - fmt.Printf("states: whether %s may be sent its new user list cannot be read: %v\n", holder, err) + // A file of the bus module's composition (novox/hq ADR 0539 clause 7), written into the bus node's want at once. + if _, err := composeModules(ctx, open, []string{holder}); err != nil { + fmt.Printf("states: the user list the states announced compose could not be composed for %s: %v\n", holder, err) return } - if why, isHeld := held[holder]; isHeld { - fmt.Printf("states: the user list the states announced compose is not sent to %s, which holds the bus: %s — "+ - "`push %s` sends it\n", holder, strings.Join(why, "; "), holder) - return - } - if _, err := sendRollout(ctx, open, []string{holder}); err != nil { - fmt.Printf("states: the user list the states announced compose could not be sent to %s: %v\n", holder, err) - return - } - fmt.Printf("states: sent %s, which holds the bus, the user list the states announced compose\n", holder) + fmt.Printf("states: composed for %s, which holds the bus, the user list the states announced compose\n", holder) } // notYetAnnounced are the owners some module reads the state of and that have announced nothing yet: what `status` diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index 32c9ac25..d2e940f2 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -152,18 +152,6 @@ func printStatus(asked answers) error { printPending(asked.pending, asked.pendingUnread) - if open, late := openPlans(asked.plans, time.Now(), asked.tierBounds); len(open) > 0 { - fmt.Printf("%d plan(s) open", len(open)) - if late > 0 { - fmt.Printf(", %d in a tier past its bound", late) - } - fmt.Println(":") - for _, p := range open { - fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.tierBounds.of(p.Repository))) - } - fmt.Println() - } - if len(behind) > 0 { var names []string for m := range behind { @@ -212,10 +200,6 @@ func printStatus(asked answers) error { fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", len(never), strings.Join(never, ", ")) } - for _, n := range sortedKeysOf(stringsKeys(asked.recordedWaits)) { - fmt.Printf(" %s is behind, waiting for a person: %s — `push %s` sends it\n", n, - strings.Join(asked.recordedWaits[n], "; "), n) - } fmt.Printf("\n `push --behind` sends them\n\n") } @@ -354,7 +338,6 @@ func printStatus(asked answers) error { // What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not // break "all well"; a record that could not be read does. - printLastSends(nodes, asked.lastSent, asked.lastSentUnread) if adopted := adoptedNodes(nodes); len(adopted) > 0 { // Said, because nothing forces the flip: a node left adopted is visible here rather than @@ -455,7 +438,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { // ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer // the private network is built from and should say nothing else; a machine that does not // resolve is already in refused, and is passed over here. - plans := map[string]planned{} for _, n := range out.nodes { plan, settings, err := resolveNode(ctx, open, n.Name) if err != nil { @@ -464,7 +446,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { } return answers{}, err } - plans[n.Name] = planned{plan, settings} // And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it. if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 { if out.leftOut == nil { @@ -490,19 +471,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { out.pendingUnread = err.Error() err = nil } - // And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through - // the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said. - if out.lastSent, err = inv.LastSends(ctx); err != nil { - out.lastSentUnread = err.Error() - err = nil - } - out.plans, err = inv.RecentPlans(ctx, 5) - if err != nil { - return answers{}, err - } - // Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219). - // And how long a tier may take, the bound a plan is late at (novox/hq issue 296). - out.tierBounds = readTierBounds(ctx, inv, time.Now()) // And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus // to read the log from; a process with none has no log to count. if _, onBus := broker.BusAddress(); onBus == nil { @@ -521,49 +489,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { out.heals = countHeals(heals) } - // And which machines are not running what the mesh would send them. The same question as a - // module being behind its source, one level down: that one says the catalogue is out of date, - // this one says a machine is — and only the second has anybody's change waiting in it. - // - // **One machine that cannot be resolved must not take the answer away from every other** - // (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub - // is the blocked machine has no hub — which used to come back here as a refusal, so `status` - // said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The - // reason is kept and reported as data; every question that does not depend on it is still - // answered. - would, err := wouldSendFrom(ctx, open, out.nodes, plans) - if err != nil { - out.network = err.Error() - would = map[string]string{} - } - out.waiting, err = inv.Waiting(ctx, would) - if err != nil { - return answers{}, err - } - // And, of those, what waits for a person's push (novox/hq ADR 0377 §6): a recorded module's move that went out - // with its walk raises nothing here. - if len(out.waiting) > 0 { - f, err := readMoveFacts(ctx, inv) - if err != nil { - return answers{}, err - } - for _, m := range out.waiting { - if m.Never { - continue - } - held, err := recordedWaiting(ctx, inv, f, m.Node) - if err != nil { - return answers{}, err - } - if len(held) > 0 { - if out.recordedWaits == nil { - out.recordedWaits = map[string][]string{} - } - out.recordedWaits[m.Node] = held - } - } - } - out.reported, err = inv.LastReports(ctx) + out.reported, err = lastReports(ctx, inv) if err != nil { return answers{}, err } @@ -650,8 +576,7 @@ func (a answers) well() bool { return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && - len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) && - a.lastSentUnread == "" + len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) } // pendingOpen is whether any pending assignment is still to be made. One that ended without being made is diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 790ae212..3ea128f2 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -185,22 +185,10 @@ type nudgingListener struct { } func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) { - // A declaration refused as older than the one the machine holds is counted by its writer — the - // controller epoch it claimed (novox/hq to-be 45 §6, S13) — and so is what the machine's own count - // says it refused beyond the refusals heard. now := time.Now() - if report.StaleRefusalOf() { - link.StaleRefusals.Refused(link.Refusal{Writer: link.WriterEpoch(report.Epoch), Epoch: report.Epoch, - Receiver: report.Node, At: now}) - } if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } - // A send refused for the generation it was composed from is kept on the send it refused, so S20 names - // its sender (novox/hq issue 234). - if report.OlderGeneration != nil { - heardGenerationRefusal(ctx, l.Enrolment.Inventory, report) - } // What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its // probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey. if report.Superseded == "" && report.Rekey == nil && report.Node != "" { @@ -210,11 +198,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if news { l.summary.nudge() } - // A machine's account of an apply is looked at at once (novox/hq ADR 0406 §5, issue 974): it ends the wait for the - // machine, and the look is what deletes its key, knowing the report was on that send. - if err == nil && report.Declared != "" && report.Superseded == "" && report.Rekey == nil { - askToLook() - } // What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions // raised or cleared from it. Only from an account of the machine the store took. if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" && @@ -224,11 +207,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e report.Node, herr) } } - if err == nil && l.open != nil && startedWell(report) { - // Off the report's path: replacing a given value sends the machine, and a report waits for - // nothing it caused (novox/hq ADR 0228). - go replaceGiven(context.WithoutCancel(ctx), l.open, report) - } // And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3, // issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path. if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" { diff --git a/cmd/mesh-controller/take_preview_test.go b/cmd/mesh-controller/take_preview_test.go index c7aac893..4bc24067 100644 --- a/cmd/mesh-controller/take_preview_test.go +++ b/cmd/mesh-controller/take_preview_test.go @@ -137,7 +137,7 @@ func TestTheKeptNetworkAndTheSettingsAreInThePreview(t *testing.T) { } c.settingsRefused = "forge: ports is a { port: machine-port } map" preview, _, _ = comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}}) - if !strings.Contains(preview, "SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: forge: ports") { + if !strings.Contains(preview, "SETTINGS DO NOT COMPOSE with the module's definition, so the composition leaves it out: forge: ports") { t.Errorf("settings that cannot compose are not said:\n%s", preview) } } diff --git a/cmd/mesh-controller/terminal_settings_test.go b/cmd/mesh-controller/terminal_settings_test.go index 3c8c4d8d..81435ac2 100644 --- a/cmd/mesh-controller/terminal_settings_test.go +++ b/cmd/mesh-controller/terminal_settings_test.go @@ -9,6 +9,9 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + + "github.com/novox/mesh-controller/internal/broker" ) // Where root creates and owns a module's directories, and which of the machine's paths reach its container, @@ -39,8 +42,10 @@ func atTheTerminal(t *testing.T, argv ...string) error { func runLine(t *testing.T, argv []string) error { t.Helper() - before := os.Args - defer func() { os.Args = before }() + before, named := os.Args, broker.ConnectionName + // run names every connection this process dials after the line's verb; a later test in this process would dial as + // that verb (issue 1945: a conditions test saw "mesh-controller verb settings"). + defer func() { os.Args, broker.ConnectionName = before, named }() os.Args = append([]string{"mesh-controller"}, argv...) return run() } @@ -208,8 +213,12 @@ func TestAServedKeyIsRefusedThroughAVerb(t *testing.T) { refused("a setting a trusted file asks for, through the verb", throughVerb(t, "settings", map[string]any{ "module": "power", "node": "anchor", "values": `{"lid":"ignore","x":0}`})) // And `trusted` is the catalogue's word: it never reaches the machine, whose engine parses strictly. - plan := printed(t, func() error { return atTheTerminal(t, "plan", "anchor", "--json") }) - if strings.Contains(plan, `"trusted"`) { + declared, _, err := composeQuestion(ctx, open, "anchor", compose.Reading, false) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(declared.Resources) + if strings.Contains(string(body), `"trusted"`) { t.Fatal("the declaration carries the catalogue's `trusted`") } } diff --git a/cmd/mesh-controller/testdata/declaration-ace.json.gz b/cmd/mesh-controller/testdata/declaration-ace.json.gz new file mode 100644 index 00000000..11f4a417 Binary files /dev/null and b/cmd/mesh-controller/testdata/declaration-ace.json.gz differ diff --git a/cmd/mesh-controller/testdata/declaration-g14.json.gz b/cmd/mesh-controller/testdata/declaration-g14.json.gz new file mode 100644 index 00000000..1629385a Binary files /dev/null and b/cmd/mesh-controller/testdata/declaration-g14.json.gz differ diff --git a/cmd/mesh-controller/testdata/mesh-host-module.json b/cmd/mesh-controller/testdata/mesh-host-module.json new file mode 100644 index 00000000..fcb6b77a --- /dev/null +++ b/cmd/mesh-controller/testdata/mesh-host-module.json @@ -0,0 +1,49 @@ +{ + "module": "mesh-host", + "version": "1", + "slug": "host", + "build": { + "artifacts": [ + { + "name": "host-arch", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-host", + "binary": "nox-mesh-host" + } + ] + }, + "consumes": [ + "restic.backed-up" + ], + "config": { + "operator-signers": { + "type": "string", + "description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.", + "default": "" + } + }, + "resources": [ + { + "id": "launcher", + "type": "file", + "path": "/usr/lib/nox-mesh-host/launch", + "mode": "0755", + "content": "#!/bin/sh\n# Supervise the host: start it, watch it, and decide what to do when it stops.\n#\n# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else\n# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give\n# up, when to roll back: all of it is policy, and policy in a unit file can only be read and\n# hoped for.\n#\n# **It is the witness of the host's own successor** (novox/hq to-be 45 §8, ADR 0227 rule 8). A\n# delivered host that is not the known-good one runs on trial: it must report its declaration\n# under its own build — which it marks by writing itself into known-good — within a bound. One\n# that crashes repeatedly, exits without standing aside for anything, or does not report in\n# bound is stopped, recorded in `rolled-back` with why, and the known-good one is run. The host\n# says every record on its reports, so the mesh raises it as a condition. One rollback per\n# version: a version in `rolled-back` is never started again; a newer delivery is.\n#\n# It does NOT exec the host. Exec would replace this process, and then only the init could\n# restart anything — which is the arrangement this exists to remove. The cost of staying is\n# signal handling, below.\n#\n# Everything a rollback does is one of: read a file, look at a directory, write a file. It shares\n# no code with the host and calls none of it: a binary that cannot start cannot be its own\n# recovery.\n#\n# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,\n# and -e would make it exit on the first one it is meant to handle.\nset -u\n\nSTATE_DIR=\"${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}\"\nLIBEXEC=\"${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}\"\n# The host that was placed by hand, used only when nothing has been delivered. The first host on a\n# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).\nFALLBACK=\"${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}\"\nVERSIONS=\"$LIBEXEC/versions\"\nBINARY=\"nox-mesh-host\"\nLIMIT=\"${MESH_HOST_START_LIMIT:-3}\"\nBACKOFF=\"${MESH_HOST_BACKOFF:-5}\"\n# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.\nBOUND=\"${MESH_HOST_TRIAL_BOUND:-600}\"\nTICK=\"${MESH_HOST_TRIAL_TICK:-5}\"\nGRACE=\"${MESH_HOST_STOP_GRACE:-20}\"\nONCE=\"${MESH_HOST_RUN_ONCE:-}\" # tests run one iteration; nothing else sets this\n\nATTEMPTS=\"$STATE_DIR/start-attempts\"\nHALTED=\"$STATE_DIR/halted\"\nPINNED=\"$STATE_DIR/rollback-pinned\"\nKNOWN_GOOD=\"$STATE_DIR/known-good\"\n# One line per verdict: from, to, when (seconds since the epoch), outcome, why — tab-separated, read\n# by the host (internal/upgrade) and said on its reports.\nROLLED=\"$STATE_DIR/rolled-back\"\nTRIAL=\"$STATE_DIR/trial\"\nEXPIRED=\"$STATE_DIR/trial-expired\"\n\nsay() { echo \"nox-mesh-host-launch: $*\" >&2; }\n\nnow() { date +%s; }\n\nknown_good() { tr -d '[:space:]' < \"$KNOWN_GOOD\" 2>/dev/null || true; }\n\ndelivered() { [ -n \"$1\" ] && [ -x \"$VERSIONS/$1/$BINARY\" ]; }\n\nrolled_back() { [ -s \"$ROLLED\" ] && cut -f1 \"$ROLLED\" 2>/dev/null | grep -qxF \"$1\"; }\n\n# The version a host path is: the directory it was delivered in, or nothing for the host placed by hand.\nversion_of() {\n\tcase \"$1\" in\n\t\t\"$VERSIONS\"/*/\"$BINARY\") v=\"${1#\"$VERSIONS\"/}\"; echo \"${v%/\"$BINARY\"}\" ;;\n\t\t*) echo \"\" ;;\n\tesac\n}\n\nrecord() { # from to outcome why\n\tprintf '%s\\t%s\\t%s\\t%s\\t%s\\n' \"$1\" \"$2\" \"$(now)\" \"$3\" \"$4\" >> \"$ROLLED\"\n}\n\n# Which host to run: the newest delivered one that was never rolled back — or, while a rollback's pin\n# stands, the version it pinned — or the one placed by hand when nothing has been delivered.\n#\n# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and\n# the next turn has to run what is on disk NOW — resolving this once would restart the same binary\n# for ever and the upgrade would never take.\n#\n# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was\n# described as, and those do not sort — \"1.10\" orders before \"1.9\". Ordering by name would start an\n# older host and call it an upgrade.\n#\n# **A pin stands until something newer arrives.** A version delivered after the pin was written, and\n# never rolled back, is a new build the mesh asks for: the pin goes and it runs, on trial.\npick_host() {\n\tnewest=\n\t# A directory with no executable in it is not a version: an interrupted delivery leaves one, and\n\t# running \"the newest\" would then mean running nothing.\n\tfor candidate in $(ls -1t \"$VERSIONS\" 2>/dev/null || true); do\n\t\tdelivered \"$candidate\" || continue\n\t\trolled_back \"$candidate\" && continue\n\t\tnewest=\"$candidate\"\n\t\tbreak\n\tdone\n\tif [ -s \"$PINNED\" ]; then\n\t\tpinned=\"$(tr -d '[:space:]' < \"$PINNED\" 2>/dev/null || true)\"\n\t\tif [ -n \"$newest\" ] && [ \"$newest\" != \"$pinned\" ] && [ \"$VERSIONS/$newest\" -nt \"$PINNED\" ]; then\n\t\t\tsay \"host $newest arrived after the rollback to $pinned; running it\"\n\t\t\trm -f \"$PINNED\"\n\t\telif delivered \"$pinned\"; then\n\t\t\techo \"$VERSIONS/$pinned/$BINARY\"\n\t\t\treturn 0\n\t\telse\n\t\t\tsay \"the pinned version '$pinned' is not delivered; ignoring the pin\"\n\t\tfi\n\tfi\n\tif [ -n \"$newest\" ]; then\n\t\techo \"$VERSIONS/$newest/$BINARY\"\n\t\treturn 0\n\tfi\n\techo \"$FALLBACK\"\n}\n\n# Go back from `from` to the known-good version, once, and say so. False when there is nowhere to go.\nroll_back() { # from why\n\tkg=\"$(known_good)\"\n\tif [ -z \"$1\" ] || [ \"$1\" = \"$kg\" ] || ! delivered \"$kg\"; then\n\t\treturn 1\n\tfi\n\trecord \"$1\" \"$kg\" rolled-back \"$2\"\n\t# The pin is what stops the launcher starting a version newer than known-good that is not the one\n\t# rolled back; the record is what stops it starting this one again.\n\tprintf '%s\\n' \"$kg\" > \"$PINNED\"\n\trm -f \"$TRIAL\"\n\tsay \"rolled back from host $1 to $kg: $2\"\n\treturn 0\n}\n\n# Watch a host on trial: done when it writes itself into known-good, stopped when the bound passes.\n# A subshell beside the host, so the launcher's own wait is untouched.\ntrial_watch() { # pid version deadline\n\twhile kill -0 \"$1\" 2>/dev/null; do\n\t\t[ \"$(known_good)\" = \"$2\" ] && return 0\n\t\tif [ \"$(now)\" -ge \"$3\" ]; then\n\t\t\tprintf '%s\\n' \"$2\" > \"$EXPIRED\"\n\t\t\tsay \"host $2 did not report within ${BOUND}s of starting; stopping it\"\n\t\t\tkill -TERM \"$1\" 2>/dev/null\n\t\t\twaited=0\n\t\t\twhile kill -0 \"$1\" 2>/dev/null && [ \"$waited\" -lt \"$GRACE\" ]; do\n\t\t\t\tsleep 1\n\t\t\t\twaited=$((waited + 1))\n\t\t\tdone\n\t\t\tkill -KILL \"$1\" 2>/dev/null\n\t\t\treturn 0\n\t\tfi\n\t\tsleep \"$TICK\"\n\tdone\n}\n\nchild=\nwatcher=\nstopping=\n\n# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that\n# exits while its child is still running leaves the host to be killed rather than to stop, and\n# an apply interrupted that way is exactly the half-configured machine this project is about.\non_term() {\n\tstopping=yes\n\tif [ -n \"$child\" ]; then\n\t\tsay \"stopping: passing the signal to the host\"\n\t\tkill -TERM \"$child\" 2>/dev/null\n\tfi\n}\ntrap on_term TERM INT\n\nmkdir -p \"$STATE_DIR\"\n# What this launcher is, so a delivered successor of it is run at the next clean exit rather than at\n# the next boot.\nSELF=\"$(cksum < \"$0\" 2>/dev/null || true)\"\n\nwhile :; do\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -e \"$HALTED\" ]; then\n\t\tsay \"halted: $(cat \"$HALTED\" 2>/dev/null || echo 'reason not recorded')\"\n\t\tsay \"not starting the host. this node needs a person.\"\n\t\texit 0\n\tfi\n\n\t# Consecutive failed starts, not starts. Cleared by the host itself when it reports, which is\n\t# the only evidence either this or known-good has.\n\t#\n\t# Read the FIRST FIELD, then insist it is a plain integer.\n\t#\n\t# Stripping whitespace instead concatenates, and that is not hypothetical: a counter\n\t# holding \"1 2\" became \"12\", past the limit, so a healthy node rolled itself back. An\n\t# unreadable counter must fail towards \"start normally\", never towards \"give up\".\n\tcount=0\n\tif [ -s \"$ATTEMPTS\" ]; then\n\t\tread -r count _ < \"$ATTEMPTS\" 2>/dev/null || count=0\n\tfi\n\tcase \"${count:-}\" in\n\t\t'' | *[!0-9]*) count=0 ;;\n\tesac\n\n\tHOST=\"$(pick_host)\"\n\tversion=\"$(version_of \"$HOST\")\"\n\n\tif [ \"$count\" -ge \"$LIMIT\" ]; then\n\t\tif roll_back \"$version\" \"it failed $count times in a row\"; then\n\t\t\t# Fresh count for the version now run: it deserves its own attempts, and without this\n\t\t\t# it inherits a count already over the limit and halts at once. The variable too, not\n\t\t\t# only the file: resetting one and not the other made the next failure count from the\n\t\t\t# OLD value — so the rolled-back version got one attempt instead of three.\n\t\t\tcount=0\n\t\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\t\tcontinue\n\t\tfi\n\t\tkg=\"$(known_good)\"\n\t\tif [ -n \"$kg\" ] && { [ \"$version\" = \"$kg\" ] || [ -z \"$version\" ]; } && [ -s \"$ROLLED\" ]; then\n\t\t\t# Already gone back, and what it went back to fails as well: this is the machine and\n\t\t\t# not a binary. Rolling back again would flap between two versions for ever.\n\t\t\tsay \"the host failed $count times after a rollback. the version it went back to does\"\n\t\t\tsay \"not start either, so this is the machine and not the binary.\"\n\t\t\trecord \"${version:-placed-by-hand}\" \"\" halted \"the version rolled back to failed $count times as well\"\n\t\t\tprintf 'rolled back and still failing\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\t\t# Nothing to go back to: no host has ever reported here, or the one that did was placed by\n\t\t# hand and is not delivered. An installation failure rather than an upgrade's — said, and\n\t\t# tried again slowly, never guessed at.\n\t\tsay \"the host failed $count times and there is no delivered known-good version to go back to.\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\tfi\n\n\tif [ ! -x \"$HOST\" ]; then\n\t\tsay \"no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable.\"\n\t\tprintf 'no host binary\\n' > \"$HALTED\"\n\t\texit 0\n\tfi\n\n\t# **On trial**: a delivered version that is not the known-good one, while a known-good one is\n\t# delivered to go back to. The trial starts when this version was first started, and survives\n\t# this launcher restarting.\n\ttrial=\n\tdeadline=\n\tkg=\"$(known_good)\"\n\tif [ -n \"$version\" ] && [ \"$version\" != \"$kg\" ] && delivered \"$kg\"; then\n\t\ttrial=yes\n\t\tstarted=\n\t\tif [ -s \"$TRIAL\" ]; then\n\t\t\tread -r on since _ < \"$TRIAL\" 2>/dev/null || on=\n\t\t\t[ \"${on:-}\" = \"$version\" ] && started=\"${since:-}\"\n\t\tfi\n\t\tcase \"$started\" in\n\t\t\t'' | *[!0-9]*) started=\"$(now)\"; printf '%s %s\\n' \"$version\" \"$started\" > \"$TRIAL\" ;;\n\t\tesac\n\t\tdeadline=$((started + BOUND))\n\t\tif [ \"$(now)\" -ge \"$deadline\" ]; then\n\t\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\" && continue\n\t\tfi\n\telse\n\t\trm -f \"$TRIAL\"\n\tfi\n\n\trm -f \"$EXPIRED\"\n\tsay \"running $HOST${trial:+ (on trial until it reports)}\"\n\t\"$HOST\" run &\n\tchild=$!\n\twatcher=\n\tif [ -n \"$trial\" ]; then\n\t\ttrial_watch \"$child\" \"$version\" \"$deadline\" &\n\t\twatcher=$!\n\tfi\n\tstatus=0\n\twait \"$child\" || status=$?\n\tif [ -n \"$stopping\" ]; then\n\t\t# The signal interrupted the wait, not the host: it was told, and is finishing what it was\n\t\t# doing. Waited for, so the service manager does not kill it half way through an apply.\n\t\twait \"$child\" 2>/dev/null\n\tfi\n\tchild=\n\tif [ -n \"$watcher\" ]; then\n\t\tkill \"$watcher\" 2>/dev/null\n\t\twait \"$watcher\" 2>/dev/null\n\t\twatcher=\n\tfi\n\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -n \"$trial\" ] && [ -s \"$EXPIRED\" ] && [ \"$(cat \"$EXPIRED\" 2>/dev/null)\" = \"$version\" ]; then\n\t\trm -f \"$EXPIRED\"\n\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\tcontinue\n\tfi\n\n\t# A signal the host did not survive, and we are not shutting down: treat it as a crash.\n\tcase \"$status\" in\n\t\t0)\n\t\t\t# Exited cleanly. That is how the host stands aside for a new binary after an\n\t\t\t# upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk.\n\t\t\t#\n\t\t\t# Deliberately NOT counted, and this is the whole reason the counter is\n\t\t\t# incremented here rather than before the start: counting attempts meant a host\n\t\t\t# that upgraded itself three times rolled itself back, having worked perfectly\n\t\t\t# every time.\n\t\t\t#\n\t\t\t# **Unless it stood aside for nothing.** A host on trial that exits cleanly while the\n\t\t\t# same host is still the one to run has not stood aside for a successor: it has\n\t\t\t# stopped, and a host that stops at once would otherwise loop here for ever unseen.\n\t\t\tif [ -n \"$trial\" ] && [ \"$(pick_host)\" = \"$HOST\" ] && [ \"$(known_good)\" != \"$version\" ]; then\n\t\t\t\tsay \"host $version exited cleanly on trial with nothing newer to stand aside for\"\n\t\t\telse\n\t\t\t\t# A delivered successor of this launcher runs from here on, not from the next boot.\n\t\t\t\tif [ -n \"$SELF\" ] && [ \"$(cksum < \"$0\" 2>/dev/null || true)\" != \"$SELF\" ]; then\n\t\t\t\t\tsay \"the launcher was replaced; running the new one\"\n\t\t\t\t\texec \"$0\"\n\t\t\t\tfi\n\t\t\t\tsay \"the host exited cleanly; starting it again\"\n\t\t\t\tcontinue\n\t\t\tfi\n\t\t\t;;\n\tesac\n\n\tcount=$((count + 1))\n\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\n\tsay \"the host exited $status ($count consecutive); restarting in ${BACKOFF}s\"\n\t[ -n \"$ONCE\" ] && exit \"$status\"\n\tsleep \"$BACKOFF\"\ndone\n" + }, + { + "id": "next", + "type": "archive", + "artifact": "host-arch", + "path": "/usr/lib/nox-mesh-host/versions/${version}" + }, + { + "id": "operator-keys", + "type": "file", + "path": "/var/lib/mesh-host/operator-keys", + "mode": "0644", + "content": "${config:operator-signers}\n" + } + ] +} diff --git a/cmd/mesh-controller/tier_send_integration_test.go b/cmd/mesh-controller/tier_send_integration_test.go deleted file mode 100644 index 5ab4a20f..00000000 --- a/cmd/mesh-controller/tier_send_integration_test.go +++ /dev/null @@ -1,408 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -func aTierMesh(t *testing.T, modules ...string) *tierMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - tm := &tierMesh{open: open, modules: modules} - tm.keeper, _ = withConditionsInMemory(t) - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: tm.keeper, teller: &conditions.Told{}} - t.Cleanup(func() { doctorFrom = was }) - - register := func(m, commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - first := map[string]string{} - for _, m := range modules { - for i, commit := range []string{"c1", "c2"} { - at := time.Now().Add(-2 * time.Hour) - if commit == "c2" { - at = time.Now().Add(-time.Minute) - } - b := inventory.Build{ID: fmt.Sprintf("build-%s-%d", m, i+1), Module: m, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register(m, "c1", time.Now().Add(-2*time.Hour)) - first[m] = "c1" - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", first); err != nil { - t.Fatal(err) - } - } - for _, m := range modules { - register(m, "c2", time.Now().Add(-time.Minute)) - } - - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - tm.sent = append(tm.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range tm.modules { - builds[m] = current[m].Commit - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, judged: true, - open: append([]conditions.Condition(nil), tm.conds...)} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - f.served[r.Node] = served{runtime: true, tools: map[string]bool{}} - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - states := map[string]*inventory.PlanModule{} - for _, m := range modules { - states[m] = &inventory.PlanModule{State: "built", BuiltAt: &built, Commit: "c2", Build: "build-" + m + "-2"} - } - plan := inventory.Plan{ID: "plan-tier", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{modules}, Modules: states} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return tm -} - -func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - var modules []string - for i := 1; i <= 12; i++ { - modules = append(modules, fmt.Sprintf("m%02d", i)) - } - tm := aTierMesh(t, modules...) - ctx := t.Context() - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the whole tier", tm.sent) - } - p := tm.plan(t) - lead := p.Modules["m01"] - if lead.Gate == nil || len(lead.Gate.Carried) != len(modules) || lead.GatedBy != "" { - t.Fatalf("the gate on the first module does not judge the whole send: %+v", lead.Gate) - } - for _, m := range modules { - s := p.Modules[m] - if s.FirstAt == nil || !reflect.DeepEqual(s.First, []string{"anchor"}) || s.Previous != "c1" { - t.Fatalf("%s's first send is not recorded, or what anchor ran before is not c1: %+v", m, s) - } - if m != "m01" && s.GatedBy != "m01" { - t.Fatalf("%s is not gated by the send's gate: %+v", m, s) - } - } - - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: one send per machine for the tier", tm.sent) - } - if p.State != inventory.PlanDone { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - for _, m := range modules { - if g := p.Modules[m].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Fatalf("%s's gate: %+v", m, g) - } - if v, found, err := tm.open.inventory.GateOf(ctx, "build-"+m+"-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's pass was not kept: %+v %v %v", m, v, found, err) - } - } -} - -func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) - ctx := t.Context() - inv := tm.open.inventory - // The node tools are held inside their witness window, which the node-engine keeps while they settle: their - // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). - gateWitnessWindow = time.Hour - - advancePlans(ctx, tm.open) - if len(tm.sent) != 1 { - t.Fatalf("sent %v", tm.sent) - } - tm.raise("anchor", kindCoreBehind, "anchor runs the node tools it was last sent, not yet reported applied, "+ - "and no plan is rolling them out") - gateEvery = 0 - advancePlans(ctx, tm.open) - g := tm.plan(t).Modules["app1"].Gate - if g.Verdict != "" || !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("inside the settle window the gate is %+v: only the node tools wait on it", g) - } - - gateBound = -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g = p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || g.Verdict != inventory.GateFailed || - !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - if current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("the node tools were not put back: %s", current[broker.RuntimeModule].Commit) - } - for _, m := range []string{"app1", "app2"} { - if current[m].Commit != "c2" { - t.Fatalf("%s was put back for the node tools' condition: %s", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Fatalf("%s's build was marked failed for the node tools' condition", m) - } - } - // Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318 - // review): it is not blamed, and not left without a verdict. - if !strings.Contains(p.Note, "app1 passed on its own and is kept") { - t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's own pass was not kept: %+v", v) - } -} - -func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "app1", "app2", "app3") - ctx := t.Context() - inv := tm.open.inventory - advancePlans(ctx, tm.open) - tm.raise("anchor", "disk-full", "anchor's root file system is full") - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g := p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || !strings.Contains(g.Why, "anchor as a whole") || - !reflect.DeepEqual(g.Failing, []string{"app1", "app2", "app3"}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - for _, m := range []string{"app1", "app2", "app3"} { - if current[m].Commit != "c1" { - t.Fatalf("%s was not put back with the send: %s", m, current[m].Commit) - } - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the first send and one send putting all of it back, and nothing to laptop", tm.sent) - } - if !strings.Contains(p.Note, "put back app1 to c1, app2 to c1, app3 to c1 on anchor, in one send") { - t.Fatalf("the plan does not say what was put back: %s", p.Note) - } -} - -func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // An earlier send carried c2 to anchor before the plan's own. - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-early", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if p := g.plan(t); p.Modules["app"].Previous != "c2" { - t.Fatalf("what anchor ran before the send: %+v", p.Modules["app"]) - } - g.health["anchor"] = healthNotYet - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || gate.Rollback != gateUnchanged || - !strings.Contains(p.Note, "app was left as it was") { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, gate) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a build its send changed nothing of was marked failed") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("the module was moved: %s", current["app"].Commit) - } - open, _ := g.keeper.Open(ctx) - for _, c := range open { - if c.Kind == kindRollbackFailed || c.Kind == kindRolledBack { - t.Fatalf("said as a rollback: %+v", c) - } - } - - everyKeyDone(t) // the retried module is taken from a done key - g.health["anchor"] = healthGood - said, err := retryPlan(ctx, g.open, "plan-gate") - if err != nil { - t.Fatalf("a plan stopped at a gate that judged no build was not retried: %v", err) - } - p = g.plan(t) - s := p.Modules["app"] - if p.State != inventory.PlanBuilding || s.State != "asked" || !s.Key || s.FirstAt != nil || s.Gate != nil { - t.Fatalf("retried as %q: the plan is %s, app %+v", said, p.State, s) - } -} - -func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { - t.Run("gate kept on "+order[0], func(t *testing.T) { - tm := aTierMesh(t, order...) - ctx := t.Context() - inv := tm.open.inventory - tierFacts := gatherGateFacts - var seen []string - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := tierFacts(ctx, open, component) - current, _ := open.inventory.CurrentBuilds(ctx) - seen = append(seen, current[broker.RuntimeModule].Commit) - if current[broker.RuntimeModule].Commit == "c2" { - f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, - Outcome: lease.OutcomeRolledBack, From: "c2", To: "c1", At: time.Now(), Why: "the node tools did not answer"}} - } - return f, err - } - // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the - // judgings that find the node tools broken, and is judged to its own pass after them (the hold the - // settle time gave before novox/hq ADR 0352 counted readings from the send). - gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second - for i := 0; i < 20 && len(seen) < 2; i++ { - advancePlans(ctx, tm.open) - } - if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { - t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at once", seen) - } - if failed, _ := inv.GateFailed(ctx, "build-"+broker.RuntimeModule+"-2"); !failed { - t.Fatal("the node tools' build is not marked failed at once") - } - if p := tm.plan(t); p.State == inventory.PlanFailed { - t.Fatalf("the plan failed at once: %s; want it judging app1 first", p.Note) - } - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, tm.open) - if tm.plan(t).State == inventory.PlanFailed { - break - } - time.Sleep(20 * time.Millisecond) - } - p := tm.plan(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the plan is %s: %s; want failed, for the node tools", p.State, p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's verdict: %+v; want its own pass (plan: %s)", v, p.Note) - } - if current, _ := inv.CurrentBuilds(ctx); current["app1"].Commit != "c2" || current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("registered app1 %s, node tools %s; want c2 and c1", current["app1"].Commit, - current[broker.RuntimeModule].Commit) - } - }) - } -} - -func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, broker.RuntimeModule, "app1") - ctx := t.Context() - // One reading counted per advance, as every test with a gate left open does: a gate judged with no time - // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateWitnessWindow = time.Hour, time.Hour - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p := tm.plan(t) - lead := p.Modules[broker.RuntimeModule].Gate - if lead == nil { - lead = p.Modules["app1"].Gate - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { - t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) - } - gateEvery, gateWitnessWindow = 0, 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) - } - for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || - !strings.Contains(g.Why, "the tool runner's rule")) { - t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) - } - } -} diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go deleted file mode 100644 index 188ec090..00000000 --- a/cmd/mesh-controller/tier_send_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A plan's tier is sent to each machine once, and judged there by one gate (novox/hq issue 281). -// -// On 2026-10-06 a merge rebuilt about seventy modules in one tier, and the plan sent the first machine -// one module at a time — one whole declaration per module, seconds apart. The node-engine set aside one -// declaration for the next, the node tools never settled, the mesh said the machine's core was behind, -// and the gate failed the module it happened to be judging for that machine-level condition; it then -// could not put the module back, having read as "the build it ran before" the very build an earlier send -// of the same tier had already carried there. - -// tierMesh is a mesh with every module of one tier running on anchor and laptop at build c1, a newer -// build c2 of each registered, a plan whose one tier built them all, and every send recorded and -// answered: the machine applies what it is sent and reports it. A judging reads the store's reports, the -// node tools as answering everywhere, and the conditions the test opens. -type tierMesh struct { - open *stores - modules []string - sent [][]string - conds []conditions.Condition - keeper *conditions.Keeper -} - -func (tm *tierMesh) plan(t *testing.T) inventory.Plan { - t.Helper() - p, err := tm.open.inventory.PlanByID(t.Context(), "plan-tier") - if err != nil { - t.Fatal(err) - } - return p -} - -// raise opens a condition about a machine itself, raised now: after the send. -func (tm *tierMesh) raise(machine, kind, summary string) { - tm.conds = append(tm.conds, conditions.Condition{Key: "machine." + machine + "." + kind, Kind: kind, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, Machine: machine}, - Raised: time.Now().Add(time.Second), Summary: summary, Source: "D10"}) -} - -// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one -// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. - -// The core behind about the node tools, while the node tools are themselves among what was sent and -// settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the -// modules sent with them: at the bound only the node tools are put back. - -// A machine-level condition about anything else holds the machine back as a whole: everything the send -// moved there fails together, once, with that reason — not one module for it. - -// The machine-level conditions, sorted: one naming a module is that module's; the core behind is the -// core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole. -func TestTheConditionsAboutAMachineItself(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - since := time.Now().Add(-time.Minute) - about := func(kind, id string) conditions.Condition { - return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(), - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: id, Machine: "anchor"}} - } - f := gateFacts{judged: true, open: []conditions.Condition{about(kindCoreBehind, "anchor")}} - if w := aboutTheMachine("anchor", []string{"app", "blueman"}, since, f); w.whole != "" || len(w.on) != 0 || len(w.facts.open) != 0 { - t.Errorf("the core behind, with no core component sent, counted against what was sent: %+v", w) - } - if w := aboutTheMachine("anchor", []string{"app", broker.RuntimeModule}, since, f); w.whole != "" || - w.on[broker.RuntimeModule] == "" || w.on["app"] != "" { - t.Errorf("the core behind, with the node tools sent, is not theirs alone: %+v", w) - } - f.open = []conditions.Condition{about("x", "anchor.app")} - if w := aboutTheMachine("anchor", []string{"app", "other"}, since, f); w.on["app"] == "" || w.on["other"] != "" || w.whole != "" { - t.Errorf("a condition naming a module is not that module's: %+v", w) - } - f.open = []conditions.Condition{about("unreachable", "anchor")} - if w := aboutTheMachine("anchor", []string{"app"}, since, f); !strings.Contains(w.whole, "anchor as a whole") { - t.Errorf("a condition about the machine is not the machine's as a whole: %+v", w) - } - f.open[0].Raised = since.Add(-time.Hour) - if w := aboutTheMachine("anchor", []string{"app"}, since, f); w.whole != "" || len(w.facts.open) != 1 { - t.Errorf("a condition older than the send counted: %+v", w) - } -} - -// A module its first send changed nothing of — an earlier send already carried this very build there — -// fails a gate as no verdict on its build: left as it was, never marked failed, nothing said as a -// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never -// judged (the incident's "NOT put back: no build kept"). - -// **In a plan's tier path, a broken module is put back at once too** (novox/hq issue 318 review): the node -// tools' witness reverts them on the first machine. Whether the gate is kept on them or on the module beside -// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging -// the other module to its own pass, and only then fails. - -// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 -// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool -// runner's rule: one reading, not before the node-engine's witness window for it has ended. diff --git a/cmd/mesh-controller/unbound_test.go b/cmd/mesh-controller/unbound_test.go index 0114131b..a1638180 100644 --- a/cmd/mesh-controller/unbound_test.go +++ b/cmd/mesh-controller/unbound_test.go @@ -34,9 +34,6 @@ func TestPlanAndNodeListAnOptionalRequirementLeftUnbound(t *testing.T) { } } const said = "unbound influxdb-api, optional for grafana" - if shown := printed(t, func() error { return planCommand(ctx, []string{"laptop"}) }); !strings.Contains(shown, said) { - t.Errorf("plan laptop does not say influxdb-api is unbound:\n%s", shown) - } if shown := printed(t, func() error { return showNode(ctx, open, "laptop") }); !strings.Contains(shown, said) { t.Errorf("node show laptop does not say influxdb-api is unbound:\n%s", shown) } diff --git a/cmd/mesh-controller/unheld_report_test.go b/cmd/mesh-controller/unheld_report_test.go deleted file mode 100644 index e012a161..00000000 --- a/cmd/mesh-controller/unheld_report_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package main - -import ( - "bytes" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// An act says what it changed about the node it acted on, and nothing about the rest of the mesh -// (novox/hq ADR 0207): after the seat dependencies shipped, every `push ` and `assign` printed -// every node's unmet dependencies, a hundred lines around the one about the module just assigned. - -func aContainer(name string) catalogue.Manifest { - return catalogue.Manifest{Module: name, Version: "1", - Resources: []map[string]any{{"id": name, "type": "container", "image": name}}} -} - -func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) { - shelf := map[string]catalogue.Manifest{ - "web": aContainer("web"), - "docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}}, - Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}}, - } - lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"}) - if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") { - t.Errorf("meeting a dependency said %v", lines) - } - // Taking the dependent off says nothing: the dependency went with its module. - if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 { - t.Errorf("unassigning the dependent said %v", lines) - } -} - -func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) { - unheld := map[string][]catalogue.Unheld{ - "anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}}, - "laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat}, - {Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}}, - } - var named bytes.Buffer - reportUnheldPushed(&named, true, []string{"laptop"}, unheld) - got := named.String() - if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") || - !strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") { - t.Errorf("a named push said:\n%s", got) - } - var all bytes.Buffer - reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld) - want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n" - if all.String() != want { - t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want) - } -} - -// The test gives logUnheld its own writer and memory, so a repeat in one process and the -// package's other tests see neither the last run's memory nor a swapped os.Stderr (novox/hq issue 486). -func TestOnlyTheServingControllerLogsEachChange(t *testing.T) { - var out bytes.Buffer - saved, savedOn := unheldLog, logUnheldChanges - unheldLog, logUnheldChanges = newUnheldLogger(&out), false - t.Cleanup(func() { unheldLog, logUnheldChanges = saved, savedOn }) - read := func(f func()) string { - out.Reset() - f() - return out.String() - } - u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}} - if got := read(func() { logUnheld("n1", u) }); got != "" { - t.Errorf("a command logged:\n%s", got) - } - logUnheldChanges = true - if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") { - t.Errorf("the serving controller did not log a change:\n%s", got) - } - if got := read(func() { logUnheld("n1", u) }); got != "" { - t.Errorf("an unchanged report was logged again:\n%s", got) - } - if got := read(func() { logUnheld("n1", nil) }); !strings.Contains(got, "every seat its modules depend on is held") { - t.Errorf("a report that became none was not logged:\n%s", got) - } -} diff --git a/cmd/mesh-controller/unseen.go b/cmd/mesh-controller/unseen.go index 1ed42bcd..53cc2acd 100644 --- a/cmd/mesh-controller/unseen.go +++ b/cmd/mesh-controller/unseen.go @@ -1,18 +1,8 @@ package main import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "os" "reflect" "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" ) // No change to a machine takes effect unseen (novox/hq ADR 0217, to-be 44). @@ -71,275 +61,3 @@ func leaves(m map[string]any, prefix string) map[string]any { } return out } - -// ---- 2. what a push will change ----------------------------------------------------------------- - -// sentResource is what is kept of one resource a machine was sent: enough to say what changed and -// whether a container's data moved, and nothing that could carry a secret — a file's content is -// kept as a digest, never as text (the record lands in the store's own backups). -type sentResource struct { - ID string `json:"id"` - Type string `json:"type"` - // Digest is of the whole resource as it was sent. - Digest string `json:"digest"` - // Fields is each field's digest, so a change can be named by field. - Fields map[string]string `json:"fields"` - // Volumes is a container's mounts as sent — paths, which are not secrets, and what a moved - // data directory is read from. - Volumes []string `json:"volumes,omitempty"` - // Image is a container's image as sent — a reference, not a secret — so the machine can be told - // which images its last declaration named, and keep them (novox/hq ADR 0251, `images`). Absent in - // a summary kept before it was. - Image string `json:"image,omitempty"` -} - -// summarize is what is kept of a declaration body: its resources, in the order sent. -func summarize(body []byte) ([]sentResource, error) { - var envelope struct { - Resources []map[string]any `json:"resources"` - } - if err := json.Unmarshal(body, &envelope); err != nil { - return nil, fmt.Errorf("a declaration that is not one: %w", err) - } - out := make([]sentResource, 0, len(envelope.Resources)) - for _, r := range envelope.Resources { - s := sentResource{ID: fmt.Sprint(r["id"]), Type: fmt.Sprint(r["type"]), Digest: digestValue(r), - Fields: map[string]string{}} - for k, v := range r { - s.Fields[k] = digestValue(v) - } - if s.Type == "container" { - if image, ok := r["image"].(string); ok { - s.Image = image - } - if vols, ok := r["volumes"].([]any); ok { - for _, v := range vols { - s.Volumes = append(s.Volumes, fmt.Sprint(v)) - } - } - } - out = append(out, s) - } - return out, nil -} - -func digestValue(v any) string { - raw, _ := json.Marshal(v) - sum := sha256.Sum256(raw) - return hex.EncodeToString(sum[:8]) -} - -// changedResource is one resource sent differently, with the fields that differ. -type changedResource struct { - ID, Type string - Fields []string -} - -// sentDiff is what would be sent now against what was sent last, by resource id. -type sentDiff struct { - Added, Removed []string - Changed []changedResource -} - -func (d sentDiff) empty() bool { - return len(d.Added) == 0 && len(d.Removed) == 0 && len(d.Changed) == 0 -} - -func diffSent(before, after []sentResource) sentDiff { - was := map[string]sentResource{} - for _, r := range before { - was[r.ID] = r - } - var d sentDiff - seen := map[string]bool{} - for _, r := range after { - seen[r.ID] = true - old, had := was[r.ID] - if !had { - d.Added = append(d.Added, r.ID) - continue - } - if old.Digest == r.Digest { - continue - } - c := changedResource{ID: r.ID, Type: r.Type} - for k, v := range r.Fields { - if old.Fields[k] != v { - c.Fields = append(c.Fields, k) - } - } - for k := range old.Fields { - if _, kept := r.Fields[k]; !kept { - c.Fields = append(c.Fields, k) - } - } - sort.Strings(c.Fields) - d.Changed = append(d.Changed, c) - } - for _, r := range before { - if !seen[r.ID] { - d.Removed = append(d.Removed, r.ID) - } - } - sort.Strings(d.Added) - sort.Strings(d.Removed) - sort.Slice(d.Changed, func(a, b int) bool { return d.Changed[a].ID < d.Changed[b].ID }) - return d -} - -// writeDiff says a diff the way a person reads one: what is added, removed and changed, and a -// changed container's fields — a container sent differently is a container recreated. -func writeDiff(w io.Writer, node string, d sentDiff, moves []dataMove) { - if d.empty() { - fmt.Fprintf(w, "%s: nothing would change — it was last sent exactly this\n", node) - return - } - fmt.Fprintf(w, "%s would change:\n", node) - for _, id := range d.Added { - fmt.Fprintf(w, " + %s\n", id) - } - for _, id := range d.Removed { - fmt.Fprintf(w, " - %s\n", id) - } - for _, c := range d.Changed { - what := "changed" - if c.Type == "container" { - what = "recreated" - } - fmt.Fprintf(w, " ~ %s %s: %s\n", c.ID, what, strings.Join(c.Fields, ", ")) - } - writeMoves(w, node, moves) -} - -// ---- 3. a running module's data moving ------------------------------------------------------------ - -// dataMove is a container keeping a mount at the same place inside it with a different directory -// on the machine behind it: its data moving — or, as on 2026-10-05, a module about to start on an -// empty directory because the placement that pointed at its data was lost. -type dataMove struct { - Container, Inside, From, To string -} - -// Module is the module the container belongs to: resource ids are `.`. -func (m dataMove) Module() string { - module, _, _ := strings.Cut(m.Container, ".") - return module -} - -// movesIn is every data move between what a machine was sent and what it would be sent. A new -// container, a mount added or dropped and a changed image are not moves. -func movesIn(before, after []sentResource) []dataMove { - was := map[string]sentResource{} - for _, r := range before { - if r.Type == "container" { - was[r.ID] = r - } - } - var out []dataMove - for _, r := range after { - old, had := was[r.ID] - if r.Type != "container" || !had { - continue - } - from := mountSources(old.Volumes) - for inside, to := range mountSources(r.Volumes) { - if prev, mounted := from[inside]; mounted && prev != to { - out = append(out, dataMove{Container: r.ID, Inside: inside, From: prev, To: to}) - } - } - } - sort.Slice(out, func(a, b int) bool { - if out[a].Container != out[b].Container { - return out[a].Container < out[b].Container - } - return out[a].Inside < out[b].Inside - }) - return out -} - -// mountSources is a container's mounts by the place inside it: `source:inside[:options]`. -func mountSources(volumes []string) map[string]string { - out := map[string]string{} - for _, v := range volumes { - parts := strings.SplitN(v, ":", 3) - if len(parts) < 2 { - continue - } - out[parts[1]] = parts[0] - } - return out -} - -func writeMoves(w io.Writer, node string, moves []dataMove) { - for _, m := range moves { - fmt.Fprintf(w, " ! %s: %s moves from %s to %s\n", m.Container, m.Inside, m.From, m.To) - } -} - -// heldMoves is the moves in a push to one machine that the operator has not said to send (`--move -// `); empty when there is nothing to hold, including a machine never sent anything before. -func heldMoves(ctx context.Context, inv *inventory.Inventory, node string, body []byte, allowed map[string]bool) ([]dataMove, error) { - prev, err := inv.SentSummary(ctx, node) - if err != nil || len(prev) == 0 { - return nil, err - } - var before []sentResource - if err := json.Unmarshal(prev, &before); err != nil { - // A record this version cannot read compares with nothing: holding every push for it would - // stop the mesh on a format, which is not what is at stake. Said, so it is not passed over. - fmt.Fprintf(os.Stderr, "%s: what it was last sent is kept in a form this version does not read, "+ - "so whether its data would move is not known: %v\n", node, err) - // empty-on-error: said above; the send replaces the unreadable record with one this version reads - return nil, nil - } - after, err := summarize(body) - if err != nil { - return nil, err - } - var held []dataMove - for _, m := range movesIn(before, after) { - if !allowed[m.Module()] { - held = append(held, m) - } - } - return held, nil -} - -// sayDataHeld tells the operator why a machine was not sent, and how to send it. -func sayDataHeld(w io.Writer, node string, moves []dataMove) { - modules := map[string]bool{} - for _, m := range moves { - modules[m.Module()] = true - } - var names []string - for m := range modules { - names = append(names, m) - } - sort.Strings(names) - fmt.Fprintf(w, "held %s: a running module's data would move (novox/hq ADR 0217)\n", node) - writeMoves(w, node, moves) - fmt.Fprintf(w, " if that is meant: push %s --move %s\n", node, strings.Join(names, ",")) -} - -// writePlanDiff says what sending body to a machine would change against what it was last sent. -func writePlanDiff(ctx context.Context, inv *inventory.Inventory, node string, body []byte) error { - after, err := summarize(body) - if err != nil { - return err - } - prev, err := inv.SentSummary(ctx, node) - if err != nil { - return err - } - if len(prev) == 0 { - fmt.Printf("%s: what it was last sent is not kept yet — the first push from this version keeps "+ - "it; %d resource(s) would be sent\n", node, len(after)) - return nil - } - var before []sentResource - if err := json.Unmarshal(prev, &before); err != nil { - return fmt.Errorf("%s: what it was last sent is kept in a form this version does not read: %w", node, err) - } - writeDiff(os.Stdout, node, diffSent(before, after), movesIn(before, after)) - return nil -} diff --git a/cmd/mesh-controller/unseen_integration_test.go b/cmd/mesh-controller/unseen_integration_test.go deleted file mode 100644 index e15a2acb..00000000 --- a/cmd/mesh-controller/unseen_integration_test.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" -) - -func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) { - err := pushCommand(context.Background(), nil) - if err == nil || !strings.Contains(err.Error(), "--all") { - t.Fatalf("a bare push was not refused: %v", err) - } - if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil { - t.Fatal("--all with --behind was accepted") - } -} diff --git a/cmd/mesh-controller/unseen_test.go b/cmd/mesh-controller/unseen_test.go deleted file mode 100644 index 6d8ad915..00000000 --- a/cmd/mesh-controller/unseen_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package main - -import ( - "bytes" - "reflect" - "strings" - "testing" -) - -// novox/hq ADR 0217: no change to a machine takes effect unseen. - -// The layer of 2026-10-05: a media server's node layer, and the one that replaced it, which kept -// `places` and lost three of its four directories and every other key. -var before = map[string]any{ - "puid": 1000.0, "pgid": 1000.0, - "expose": map[string]any{"32400": "anywhere"}, - "places": map[string]any{ - "config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"}, - "data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"}, - "transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"}, - }, -} - -func TestASettingThatKeepsAKeyAndLosesItsEntriesIsSaidToRemoveThem(t *testing.T) { - after := map[string]any{"places": map[string]any{ - "previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"}, - }} - added, changed, removed := settingsChange(before, after) - if !reflect.DeepEqual(added, []string{"places.previews.owner", "places.previews.path"}) { - t.Errorf("added %v", added) - } - if len(changed) != 0 { - t.Errorf("changed %v", changed) - } - for _, lost := range []string{"expose.32400", "pgid", "places.config.path", "places.data.owner", "puid"} { - found := false - for _, r := range removed { - found = found || r == lost - } - if !found { - t.Errorf("removing %s was not said: %v", lost, removed) - } - } -} - -func TestASettingThatOnlyAddsOrChangesRemovesNothing(t *testing.T) { - after := map[string]any{ - "puid": 1001.0, "pgid": 1000.0, - "expose": map[string]any{"32400": "anywhere"}, - "places": map[string]any{ - "config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"}, - "data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"}, - "transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"}, - "previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"}, - }, - } - _, changed, removed := settingsChange(before, after) - if len(removed) != 0 { - t.Errorf("an additive set was said to remove %v", removed) - } - if !reflect.DeepEqual(changed, []string{"puid"}) { - t.Errorf("changed %v", changed) - } -} - -// What was sent, as a push would send it: a media server's container and a file with a secret. -func declaration(configFrom string, extra ...string) []byte { - vols := `"` + configFrom + `:/config", "/srv/media/data:/data"` - for _, e := range extra { - vols += `, "` + e + `"` - } - return []byte(`{"declaration":1,"sequence":7,"resources":[ - {"id":"plex.server","type":"container","image":"plex@sha256:aa","volumes":[` + vols + `]}, - {"id":"plex.env","type":"file","path":"/srv/media/env","content":"TOKEN=the-secret-itself"}]}`) -} - -func summarized(t *testing.T, body []byte) []sentResource { - t.Helper() - s, err := summarize(body) - if err != nil { - t.Fatal(err) - } - return s -} - -func TestWhatIsKeptOfASendHoldsNoFileContent(t *testing.T) { - s := summarized(t, declaration("/srv/media/config")) - var kept bytes.Buffer - for _, r := range s { - kept.WriteString(r.ID + r.Type + r.Digest + strings.Join(r.Volumes, ",")) - for k, v := range r.Fields { - kept.WriteString(k + v) - } - } - if strings.Contains(kept.String(), "the-secret-itself") { - t.Fatal("a file's content was kept in the record of what was sent") - } - if len(s) != 2 || s[0].Volumes[0] != "/srv/media/config:/config" { - t.Fatalf("summary %+v", s) - } -} - -func TestADiffOfAnUnchangedMachineIsEmptyAndAChangedContainerNamesItsField(t *testing.T) { - was := summarized(t, declaration("/srv/media/config")) - if d := diffSent(was, summarized(t, declaration("/srv/media/config"))); !d.empty() { - t.Fatalf("an unchanged machine differs: %+v", d) - } - d := diffSent(was, summarized(t, declaration("/var/lib/plex/config"))) - if len(d.Changed) != 1 || d.Changed[0].ID != "plex.server" || !reflect.DeepEqual(d.Changed[0].Fields, []string{"volumes"}) { - t.Fatalf("diff %+v", d) - } - var out bytes.Buffer - writeDiff(&out, "home", d, movesIn(was, summarized(t, declaration("/var/lib/plex/config")))) - for _, want := range []string{"~ plex.server recreated: volumes", "! plex.server: /config moves from /srv/media/config to /var/lib/plex/config"} { - if !strings.Contains(out.String(), want) { - t.Errorf("diff does not say %q:\n%s", want, out.String()) - } - } -} - -// The incident: the configuration mount would move to an empty default directory. -func TestARunningContainersDataMovingIsAMoveAndAnAddedMountIsNot(t *testing.T) { - was := summarized(t, declaration("/srv/media/config")) - moves := movesIn(was, summarized(t, declaration("/var/lib/plex/config"))) - want := []dataMove{{Container: "plex.server", Inside: "/config", From: "/srv/media/config", To: "/var/lib/plex/config"}} - if !reflect.DeepEqual(moves, want) { - t.Fatalf("moves %+v", moves) - } - if moves[0].Module() != "plex" { - t.Errorf("module %q", moves[0].Module()) - } - // A mount added — the previews of 2026-10-05 — is not a move. - if m := movesIn(was, summarized(t, declaration("/srv/media/config", "/srv/pool/previews:/config/Media"))); len(m) != 0 { - t.Errorf("an added mount was held as a move: %+v", m) - } - // Nor is a container the machine never ran. - if m := movesIn(nil, was); len(m) != 0 { - t.Errorf("a first send was held as a move: %+v", m) - } -} - -func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - want []string - }{ - {"settings", map[string]any{"module": "plex", "node": "home"}, []string{"settings", "show", "plex", "--node", "home"}}, - {"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}}, - {"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}}, - {"push", map[string]any{"node": "home", "move": "plex", "why": "w"}, - []string{"push", "home", "--wait", "0s", "--move", "plex", "--why", "w"}}, - {"push", map[string]any{"why": "w"}, []string{"push", "--behind", "--wait", "0", "--why", "w"}}, - {"plan", map[string]any{"node": "home", "diff": "true"}, []string{"plan", "home", "--diff"}}, - } { - got, err := argvFor(c.verb, c.args) - if err != nil || !reflect.DeepEqual(got, c.want) { - t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want) - } - } -} - -// An argument of the guards given where it cannot take effect is refused, not passed over (issue 244): -// a move with no machine named, a replace beside a clear, a history beside values, a diff beside files. -func TestTheGuardsArgumentsAreNotPassedOver(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - }{ - {"push", map[string]any{"move": "plex", "why": "w"}}, - {"settings", map[string]any{"module": "plex", "clear": "true", "replace": "true"}}, - {"settings", map[string]any{"module": "plex", "values": "{}", "history": "true"}}, - {"plan", map[string]any{"node": "home", "files": "true", "diff": "true"}}, - } { - if argv, err := argvFor(c.verb, c.args); err == nil { - t.Errorf("%s %v was taken as %v, and an argument passed over", c.verb, c.args, argv) - } - } -} diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index d3155a4f..f17c80be 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -38,92 +37,11 @@ type following struct { // are built on. So "record it" is the absence of an action, and the only thing this has to decide // is whether to act. func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { - inv := f.open.inventory - - // The store read first, and an outage there said as one, so the announcement is held and asked - // again (novox/hq issue 083). Only here: a push that fails further down is not asked again. - decision, err := inv.UpgradeOf(ctx, u.Module) - if err != nil { - return notNow(err) - } - on, err := inv.Running(ctx, u.Module) - if err != nil { - return notNow(err) - } - if len(on) == 0 { - fmt.Printf("%s moved to %s; no machine runs it\n", u.Module, shortCommit(u.Commit)) - return nil - } - if !decision.RollOut { - // Named rather than counted, and said even though nothing happens: an upgrade that was - // deliberately not rolled out and an upgrade that was never noticed look identical in a - // log that only speaks when it acts. - fmt.Printf("%s moved to %s; %s %s behind it, and this mesh records upgrades rather than "+ - "rolling them out — `push --behind` when you want them\n", - u.Module, shortCommit(u.Commit), readableList(on), isAre(len(on))) - return nil - } - - // **A plan that holds the module rolls it out, and this does not** (novox/hq issue 249, ADR - // 0218). A merge's plan builds the module and sends it one machine first, the rest once that one - // has applied it; this announcement arrives as the build registers, and sending here too — one - // machine after another without waiting for any to apply — put the new bundle on every machine in - // the same minute, whatever the plan was waiting for. A move no plan answers (a build asked by - // hand) is still this handler's. - if plans, err := inv.OpenPlans(ctx); err != nil { - return notNow(err) - } else if id := rolledOutByAPlan(plans, u.Module); id != "" { - // Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves - // these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249). - fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+ - "`status` lists them as behind and `push --behind` sends it\n", - u.Module, shortCommit(u.Commit), id, readableList(on)) - return nil - } - // **No plan holds it: a release plan sends it** (novox/hq ADR 0236). A build asked outside a plan — a - // `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it - // now waits for a gate like any other build, and the release plan walks it through the machines, one - // at a time, each judged. - fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+ - "(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on)) + // Nothing is sent: each machine follows the wants the assigner writes for the new fingerprint (ADR 0539 §5). + fmt.Printf("%s moved to %s\n", u.Module, shortCommit(u.Commit)) return nil } -// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249): -// an announcement handled by a send whose memberships could not be issued is held and redelivered, -// rather than taken as handled with the machines left on the old version. -func askAgainOnGrants(err error) error { - if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) { - return fmt.Errorf("%w: %w", link.ErrTryAgain, err) - } - return err -} - -// rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding -// the module that has not finished sending it — or empty when none will (novox/hq issue 249). -func rolledOutByAPlan(plans []inventory.Plan, module string) string { - for _, p := range plans { - if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) { - return p.ID - } - } - return "" -} - -// readableList names machines the way a sentence does, because this is read by a person deciding -// whether an upgrade went where they expected. -func readableList(names []string) string { - switch len(names) { - case 0: - return "nothing" - case 1: - return names[0] - case 2: - return names[0] + " and " + names[1] - } - return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] -} - func shortCommit(commit string) string { if len(commit) > 8 { return commit[:8] @@ -131,125 +49,6 @@ func shortCommit(commit string) string { return commit } -func isAre(n int) string { - if n == 1 { - return "is" - } - return "are" -} - -// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR -// 0235): every module's policy and where it comes from; one module's; or a person's choice for one. -func upgradeCommand(ctx context.Context, args []string) error { - // What waits for a gate, and releasing it by a person's word (ADR 0236). - if len(args) > 0 && (args[0] == "backlog" || args[0] == "release-backlog") { - return backlogCommand(ctx, args[0], args[1:]) - } - set := flag.NewFlagSet("upgrade", flag.ContinueOnError) - together := set.Bool("together", false, - "send every machine running it at once, instead of one machine first") - why := set.String("why", "", "why this choice — required for record; kept and said with the policy") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - - if len(positionals) == 0 { - all, err := inv.Upgrades(ctx) - if err != nil { - return err - } - names := make([]string, 0, len(all)) - for n := range all { - names = append(names, n) - } - sort.Strings(names) - for _, n := range names { - u := all[n] - line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From) - if u.Why != "" { - line += ": " + u.Why - } - fmt.Println(line) - } - return nil - } - module := positionals[0] - if len(positionals) == 1 { - decision, err := inv.UpgradeOf(ctx, module) - if err != nil { - return err - } - fmt.Println(sayUpgrade(module, decision)) - return nil - } - - decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()} - switch positionals[1] { - case "roll-out", "roll": - decision.RollOut, decision.Together = true, *together - case "record": - if *together { - // Refused rather than ignored: --together only means anything for a roll-out, and - // accepting it here would store a preference that never applies and looks like it does. - return errors.New("`--together` says how to roll out, so it cannot be given with " + - "`record`, which is the choice not to") - } - if decision.Why == "" { - return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why "+ - "(novox/hq ADR 0236). Nothing was changed", module) - } - case "default": - if err := inv.ClearUpgradeOf(ctx, module); err != nil { - return err - } - decision, err := inv.UpgradeOf(ctx, module) - if err != nil { - return err - } - fmt.Println(sayUpgrade(module, decision)) - return nil - default: - return fmt.Errorf("upgrade roll-out|record|default — not %q", positionals[1]) - } - if err := inv.SetUpgradeOf(ctx, module, decision); err != nil { - return err - } - decision, err = inv.UpgradeOf(ctx, module) - if err != nil { - return err - } - fmt.Println(sayUpgrade(module, decision)) - return nil -} - -func sayUpgrade(module string, u inventory.Upgrade) string { - from := " (" + u.From - if u.By != "" { - from += ", " + u.By - } - if u.Why != "" { - from += ": " + u.Why - } - from += ")" - if !u.RollOut { - return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+ - "reported as behind until a person pushes them%s", module, from) - } - if u.Together { - return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+ - "together%s", module, from) - } - return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+ - "the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from) -} - // Announceable is every build this mesh recorded, in the shape the builder announces one. // // **The catalogue asks for this when it starts, and the answer is the graph's foundation** @@ -294,114 +93,13 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error return out, nil } -// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so -// the announcement is held rather than lost; anything else is returned as it was. -func notNow(err error) error { - if inventory.Unreachable(err) || errors.Is(err, inventory.ErrPlansBusy) { - return fmt.Errorf("%w: %w", link.ErrTryAgain, err) - } - return err -} - -// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save -// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407 -// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded. -func unsaved(err error) error { - if err == nil { - return nil - } - return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err) -} - // SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which // becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go. func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { - if err := f.hearMerge(ctx, m, time.Now().UTC()); err != nil { - return err - } - // And the merges of that trunk it did not hear, in the trunk's order (novox/hq ADR 0457): a catch-up that leaves - // something for later is no failure of this merge's hearing, and is never a reason to hear it again. - if !f.oneShot { - trunk := &caughtTrunk{Owner: m.Owner, Repo: m.Repo, Branch: m.Base} - catchUpBeside(func() { f.catchUp(context.WithoutCancel(ctx), trunk) }) - } - return nil -} - -// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and -// branch is marked as moved to the merge commit; a module the merge deleted is forgotten or said; a new module -// is asked for and sent nowhere. The names returned are what the walk builds, bases first along the graph -// (novox/hq 04-ISSUES/131). Nothing is built or pushed here: the walk asks its tiers. Called when a batch is -// cut, once per repository, with the latest merge of the repository's branch and every file the batch's -// merges of it changed. -func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) ([]string, error) { - from, packaging, already := mergeCandidates(m, entries, read) - if len(from) == 0 && len(packaging) == 0 { - // "Already built from it" and "nothing reads it" are different facts, and reading the first - // as the second sends somebody looking for a broken trigger when the mesh is up to date. - if already > 0 { - fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+ - "from it\n", m.Owner, m.Repo, m.Base, m.Commit, already) - return nil, nil - } - fmt.Print(nothingReadsIt(m.Owner+"/"+m.Repo, m)) - return nil, nil - } - // Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows - // another branch is not part of this merge, and whoever is waiting for its change should read why. - for _, e := range entries { - if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) { - fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n", - e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base) - } - } - touched, added, _ := touchedBy(from, entries, m, read) - // **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build - // seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with - // every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise. - touched, deleted := splitDeleted(touched, m) - for _, e := range deleted { - retireDeleted(ctx, inv, e, m) - } - for _, e := range touched { - if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil { - return nil, notNow(err) - } - } - // **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan - // says so, and assigning it is a person's act, which needs it registered first. - // Nothing asks its first build (novox/hq ADR 0501 §6): the builder builds every fingerprint the indexer writes, and - // the controller registers the module from its first done key (new_modules.go). - var built []string - if len(from) > 0 { - built = added - } - moved := append(append([]inventory.Entry{}, touched...), packaging...) - if len(moved) == 0 { - if len(built) > 0 { - fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+ - "registered from the builder's done key of each, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit, - strings.Join(built, ", ")) - return nil, nil - } - fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+ - "built from\n", m.Owner, m.Repo, m.Base, m.Commit) - return nil, nil - } - // Why each is in it (issue 363): the files of its build source the merge changed, or why it is read whole. - for _, e := range moved { - fmt.Printf(" %s: %s\n", e.Manifest.Module, whyMoved(e, read[e.Manifest.Module], m)) - } - for _, e := range packaging { - fmt.Printf(" %s reads %s/%s through its build context: its own source record is left where it is\n", - e.Manifest.Module, m.Owner, m.Repo) - } - var names []string - for _, e := range moved { - names = append(names, e.Manifest.Module) - } - return names, nil + // A merge plans nothing here any more: mesh-delivery merges on the Approve, the indexer writes the fingerprints and + // the builder builds them (ADR 0539 §5, §17). The controller keeps the announcement, for the build of a merge that + // moves a module between repositories to be shown the operator's Approve (ADR 0420 §7). + return keepForgeMerge(ctx, m) } // actingOnMerges keeps one merge acted on at a time (novox/hq issue 266). @@ -467,63 +165,6 @@ func lookedAtCommit(read []inventory.ReadRepository, commit string) bool { // historyMargin is how far a packaging module's last look is taken back before a merge is history for it. const historyMargin = time.Minute -// whyMoved is why a merge moves a module, as a plan says it (novox/hq ADR 0267, issue 363): the changed -// files in its build source, or why it is read whole. For a module built from the merged repository or one -// whose build context is that repository; a dependent is in a plan for what it stands on. -func whyMoved(e inventory.Entry, read []inventory.ReadRepository, m link.SourceMoved) string { - if len(m.Paths) == 0 || m.PathsTruncated { - return "read whole: the merge's changed files were not all said" - } - whole := "no build source recorded" - for _, r := range read { - if r.Own && r.Whole != "" { - whole = r.Whole - } - } - held := func(paths []string) []string { - var in []string - for _, p := range m.Paths { - if gosource.SourceHolds(paths, p) { - in = append(in, p) - } - } - return in - } - changed := func(where string, in []string) string { - return fmt.Sprintf("its build source%s changed: %d changed file(s) in it, e.g. %s", where, len(in), in[0]) - } - if sameRepository(e.Source.Repository, m) { - if own := ownSource(read); own != nil { - if in := held(own); len(in) > 0 { - return changed("", in) - } - } - dir := strings.Trim(e.Source.Path, "/") - if dir == "" { - return "read whole: " + whole + ", so every file of its repository is its build source" - } - for _, p := range m.Paths { - if inside(p, dir) { - return "read whole: " + whole + ", so its directory is its build source; e.g. " + p - } - } - return "read whole: " + whole - } - for _, r := range read { - if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) { - continue - } - if len(r.Paths) > 0 { - if in := held(r.Paths); len(in) > 0 { - return changed(" in "+m.Owner+"/"+m.Repo, in) - } - continue - } - return "read whole: " + whole + ", so every file of " + m.Owner + "/" + m.Repo + ", which its build context is, is its build source" - } - return "read whole: " + whole -} - // lookedOf is when a module packaging another repository was last looked at, as readForPlanning says. func lookedOf(read []inventory.ReadRepository) time.Time { var at time.Time @@ -535,21 +176,6 @@ func lookedOf(read []inventory.ReadRepository) time.Time { return at } -// wouldMove is the modules acting on this merge would move and rebuild — SourceMoved's judgement, made -// without acting (novox/hq issue 266). Empty for a merge already acted on: acting marks each module built -// from the repository as looked at, so the merge then reads as history for it. -// -// **The ones packaging source from it too** (novox/hq ADR 0267): with a module moved only by the files of -// its build source, a merge can move a packaging module and nothing built from the repository, and a missed -// one of those was never acted on. A packaging module's look is its newest build or plan (lookedAt), so a -// merge acted on for it reads as history once its plan is made. -func wouldMove(m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) []inventory.Entry { - from, packaging, _ := mergeCandidates(m, entries, read) - touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m, read), m) - return append(touched, packaging...) -} - // splitDeleted parts the modules a merge touched into those it changed and those whose manifest it // removed — deleted at their source (ADR 0236). func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) { @@ -572,39 +198,6 @@ func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted return kept, deleted } -// retireDeleted is what the mesh does with a module deleted at its source: its record says the merge -// was looked at, so it is not acted on again; it is forgotten where nothing holds it; where a machine -// runs it or the mesh holds something for it, that is said, and nothing is built. -func retireDeleted(ctx context.Context, inv *inventory.Inventory, e inventory.Entry, m link.SourceMoved) { - name := e.Manifest.Module - if err := inv.SourceMoved(ctx, name, m.Commit); err != nil { - fmt.Printf(" %s was deleted from %s/%s at %.8s, and that could not be recorded: %v\n", name, m.Owner, m.Repo, m.Commit, err) - } - err := inv.ForgetModule(ctx, name) - switch { - case err == nil: - fmt.Printf(" %s was deleted from %s/%s at %.8s: forgotten, nothing built\n", name, m.Owner, m.Repo, m.Commit) - case errors.Is(err, inventory.ErrStillAssigned) || errors.Is(err, inventory.ErrStillHolds): - fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; the mesh still runs or holds it, so it is "+ - "kept until a person unassigns it and `module forget %s`: %v\n", name, m.Owner, m.Repo, m.Commit, name, - firstLine(err.Error())) - default: - fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; forgetting it failed: %v\n", name, m.Owner, - m.Repo, m.Commit, err) - } -} - -// nothingReadsIt is what is said of a merge no module the mesh holds reads. **A merge into a branch other than the -// trunk is said to start no delivery** (novox/hq ADR 0401 clause 3, issue 889): every delivery is cut from a -// repository's trunk, and whoever merged into a feature branch should not look for a broken trigger. -func nothingReadsIt(repository string, m link.SourceMoved) string { - if m.Base != "" && m.Base != "main" { - return fmt.Sprintf("%s merged into %s (%.8s), not the trunk of %s; a merge into any other branch starts no "+ - "delivery\n", repository, m.Base, m.Commit, repository) - } - return fmt.Sprintf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit) -} - // sourceIs is whether a recorded source is the repository and branch a merge announced. A source on // the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from. // An empty recorded ref is the repository's default branch, which is what a merge into the base @@ -706,182 +299,10 @@ func readsDirectory(e inventory.Entry, p string) bool { return strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path) } -// unchangedOfMerge are the modules a merge changed files for under a reading without recorded build sources, but -// whose recorded build source holds none of them, each with why (novox/hq issue 611): not built, because the build -// the mesh holds is theirs (ADR 0267 §5). That reading — a delivery plan made while another walk built the module -// (issue 606) — would have moved them; the walk says they matched what runs rather than leaving them unnamed. -// A module built from the repository whose directory, or whole repository when built from its root, holds a -// changed file, and a module packaging the repository's branch, are each read so. A merge whose changed files -// were not all said moves every one of them: none is said here. Without acting. -func unchangedOfMerge(m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) map[string]string { - if len(m.Paths) == 0 || m.PathsTruncated { - return nil - } - from, _, _ := mergeCandidates(m, entries, read) - touched, _, _ := touchedBy(from, entries, m, read) - out := map[string]string{} - for _, e := range from { - name := e.Manifest.Module - if ownSource(read[name]) == nil || slices.ContainsFunc(touched, func(t inventory.Entry) bool { - return t.Manifest.Module == name - }) { - continue - } - var in []string - for _, p := range m.Paths { - if readsDirectory(e, p) { - in = append(in, p) - } - } - if len(in) > 0 { - out[name] = fmt.Sprintf("its build source holds none of the %d changed file(s) in its directory (e.g. %s): "+ - "the build the mesh holds is its build", len(in), in[0]) - } - } - for _, e := range entries { - name := e.Manifest.Module - if sourceIs(e.Source, m) || !packagesSaid(read[name], m) || readsFrom(read[name], m) { - continue - } - out[name] = fmt.Sprintf("its build source in %s/%s holds none of the %d changed file(s) (e.g. %s): "+ - "the build the mesh holds is its build", m.Owner, m.Repo, len(m.Paths), m.Paths[0]) - } - if len(out) == 0 { - return nil - } - return out -} - -// packagesSaid is whether a module's newest trunk build said the build source it read in a merge's repository and -// branch, as a context beside its own. -func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool { - for _, r := range read { - if !r.Own && sameRepository(r.Repository, m) && followsBase(r.Ref, m.Base) && len(r.Paths) > 0 { - return true - } - } - return false -} - -// staleIn is the modules whose recorded build source a plan has overtaken (novox/hq ADR 0267): a plan still -// working that has yet to build one, or a plan made after that build which never built it — failed, stopped -// or superseded. What such a module is built from is changing, or changed without a build to say so: a merge -// that added an import to it, and a later one changing only what that import names, would otherwise move -// nothing. Each is read whole, as before, until a build of it works again. -// -// Each is answered with the plan that overtook it, for saying why it is read whole. -func staleIn(read map[string][]inventory.ReadRepository, plans []inventory.Plan) map[string]string { - stale := map[string]string{} - for name, rs := range read { - var since time.Time - for _, r := range rs { - if r.Built.After(since) { - since = r.Built - } - } - for _, p := range plans { - s, in := p.Modules[name] - if !in || (s != nil && (s.State == "built" || s.State == planDeleted)) { - continue - } - if p.Open() || p.Created.After(since) { - stale[name] = p.ID - } - } - } - return stale -} - -// lookedAt is when a merge was last acted on for a module that packages another repository's source: its -// newest build, or the newest plan that built it or is still building it, whichever is later. A build asked -// after a merge clones that repository with the merge in it, so an older merge is history for it; a plan -// that closed without building it looked at nothing. -func lookedAt(name string, read []inventory.ReadRepository, plans []inventory.Plan) time.Time { - var at time.Time - for _, r := range read { - if r.Looked.After(at) { - at = r.Looked - } - } - for _, p := range plans { - s, in := p.Modules[name] - if in && (p.Open() || (s != nil && s.State == "built")) && p.Created.After(at) { - at = p.Created - } - } - return at -} - -// readForPlanning is what each module's build read, as the planner maps a change onto it — for a merge -// acting now, the merge gate, a pull request's check, a delivery's order and the what-if alike, so planning -// and gating cannot disagree (novox/hq ADR 0238): the build sources the newest trunk builds said, but for -// the modules a plan has overtaken (staleIn), and with when each was last looked at (lookedAt). +// readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's +// check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17). func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) { - read, err := inv.ReadRepositories(ctx) - if err != nil { - return nil, err - } - // Every plan since the oldest build whose source is recorded: one made after a module's build can have - // overtaken it, however long ago, so no window of recent plans would do. - var oldest time.Time - for _, rs := range read { - for _, r := range rs { - if !r.Built.IsZero() && (oldest.IsZero() || r.Built.Before(oldest)) { - oldest = r.Built - } - } - } - plans, err := inv.PlansSince(ctx, oldest) - if err != nil { - return nil, err - } - return planningView(read, plans), nil -} - -// planningView is readForPlanning over what was read, so a test can hand it records. -func planningView(read map[string][]inventory.ReadRepository, plans []inventory.Plan) map[string][]inventory.ReadRepository { - stale := staleIn(read, plans) - out := make(map[string][]inventory.ReadRepository, len(read)) - for name, rs := range read { - looked := lookedAt(name, rs, plans) - var commits []string - for _, p := range plans { - if st, in := p.Modules[name]; in && p.Commit != "" && (p.Open() || (st != nil && st.State == "built")) { - // Every commit the walk carries (novox/hq ADR 0276): a batch's walk answers one per repository. - for _, c := range p.Carried() { - commits = append(commits, c.Commit) - } - } - } - var kept []inventory.ReadRepository - overtaken, isStale := stale[name] - for _, r := range rs { - if isStale { - if r.Own { - continue - } - r.Paths = nil - } - r.Looked, r.LookedAt = looked, commits - kept = append(kept, r) - } - if isStale { - kept = append(kept, inventory.ReadRepository{Own: true, Whole: "plan " + overtaken + " has not built it yet", - Looked: looked, LookedAt: commits}) - } - out[name] = kept - } - return out -} - -// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a -// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is -// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules. -func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved, - read map[string][]inventory.ReadRepository) []inventory.Entry { - touched, _, _ := touchedBy(candidates, known, m, read) - return touched + return inv.ReadRepositories(ctx) } // touchedBy maps a merge's changed files onto the mesh's modules — **the one place it is done**, for the @@ -989,9 +410,9 @@ type mergeReach struct { Already int // Added are the directories the change holds a new module in; Unread the changed files no build reads. Added, Unread []string - // Plan is what moves and everything that follows it along the catalogue's dependencies, tiered — - // what a merge would build. Empty when nothing moves. - Plan inventory.Plan + // Following are the modules that stand on the moved ones along the catalogue's dependencies: what a merge would + // build after them. + Following []string } // Moved are the modules the merge moves itself, by name: touched, deleted and packaging. @@ -1008,36 +429,7 @@ func (r mergeReach) Moved() []string { // Dependents are the modules the plan builds after the moved ones because they stand on them. func (r mergeReach) Dependents() []string { - moved := map[string]bool{} - for _, name := range r.Moved() { - moved[name] = true - } - var out []string - for name := range r.Plan.Modules { - if !moved[name] { - out = append(out, name) - } - } - sort.Strings(out) - return out -} - -// reachOfMerge is what a merge of these changed files into this branch would move and build, without -// acting: the merge handler's own judgement and the release plan's dependency walk. -func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge) mergeReach { - from, packaging, already := mergeCandidates(m, entries, read) - touched, added, unread := touchedBy(from, entries, m, read) - kept, deleted := splitDeleted(touched, m) - r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread} - var building []string - for _, e := range append(append([]inventory.Entry{}, kept...), packaging...) { - building = append(building, e.Manifest.Module) - } - if len(building) > 0 { - r.Plan = planOfMerge(m, building, edges) - } - return r + return r.Following } // saidModuleDirs is the directories the announcer says hold a module at the merge commit (novox/hq @@ -1066,49 +458,6 @@ func inside(path, dir string) bool { return path == dir || strings.HasPrefix(path, dir+"/") } -// anyInside is whether any of these changed files is in a directory. -func anyInside(paths []string, dir string) bool { - for _, p := range paths { - if inside(p, dir) { - return true - } - } - return false -} - -// standsOnModule is whether an entry's build stood on the named module: by what its newest build -// recorded it was handed (`artifact-store:///@…`, the module's own artifact), or -// — for a module registered from a manifest and not yet built — by the base its manifest names. -func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool { - if e.Manifest.Module == module { - return false - } - if e.Manifest.Build != nil { - for _, on := range e.Manifest.Build.On { - if on.Module == module { - return true - } - } - } - prefix := catalogue.ArtifactStoreScheme + module + "/" - for _, ref := range against[e.Manifest.Module] { - if strings.HasPrefix(ref, prefix) { - return true - } - } - return false -} - -// standsOn is whether anything in the set is built on the named module's artifacts. -func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool { - for _, e := range entries { - if standsOnModule(e, module, against) { - return true - } - } - return false -} - // isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge // with no time on it is taken as news: refusing it would silence a forge that says less. func isHistory(mergedAt string, seen time.Time) bool { @@ -1122,48 +471,178 @@ func isHistory(mergedAt string, seen time.Time) bool { return at.Before(seen) } -// dependentsOf is every catalogued module that stands on one of the moved modules, directly or -// through another dependent, and is not itself among them — in the catalogue's order, so the -// answer is the same each time. A module standing on nothing that moved is left alone: a merge -// rebuilds what it changed and what is built on top of that, not the catalogue. -func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry { - bases := map[string]bool{} - for _, e := range moved { - bases[e.Manifest.Module] = true - } - var out []inventory.Entry - taken := map[string]bool{} - for grew := true; grew; { - grew = false - for _, e := range entries { - name := e.Manifest.Module - if bases[name] || taken[name] { - continue - } - for base := range bases { - if standsOnModule(e, base, against) { - taken[name] = true - out = append(out, e) - grew = true - break - } - } - } - for _, e := range out { - bases[e.Manifest.Module] = true - } - } - return out -} - // moduleManifestFile is the file that says what a module is, in its directory (the build seat's // ManifestName). const moduleManifestFile = "module.json" -// deletedAtSource is whether a build failed because its module's manifest is not at its source any -// more — the build seat's own words (internal/builder) — which a merge that deleted the module causes -// when its announcer did not say which files went (ADR 0236). Such a module is not a failure of the plan. -func deletedAtSource(failed string) bool { - return strings.Contains(failed, "has no "+moduleManifestFile+" at ") && - strings.Contains(failed, "so there is nothing saying what it is") +// reachOfMerge is what a merge of these changed files into this branch would move, without acting: a pull request's +// reach, as the check reckons it. +func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository, + edges []inventory.Edge) mergeReach { + from, packaging, already := mergeCandidates(m, entries, read) + touched, added, unread := touchedBy(from, entries, m, read) + kept, deleted := splitDeleted(touched, m) + r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread} + moved := r.Moved() + if len(moved) > 0 { + in := map[string]bool{} + for _, name := range moved { + in[name] = true + } + for _, name := range reachableFrom(moved, edges) { + if !in[name] { + r.Following = append(r.Following, name) + } + } + } + return r +} + +// reachableFrom is the moved modules plus everything that depends on them, through every layer: +// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine +// is not changed by a new build machine, so a built-by edge orders and gates a plan and never +// widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change. +func reachableFrom(moved []string, edges []inventory.Edge) []string { + in := map[string]bool{} + for _, m := range moved { + in[m] = true + } + for grew := true; grew; { + grew = false + for _, e := range edges { + // Built-by and worker-of order a plan; neither widens it. A new build machine changes + // nothing it builds, and a new controller changes nothing about the holder it orders. A + // packages edge, read from a record made before novox/hq ADR 0267, widens nothing either: + // a shared file moves each module whose build source holds it, directly. + if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages || + e.Kind == edgeChangeOrder { + continue + } + if in[e.To] && !in[e.From] { + in[e.From] = true + grew = true + } + } + } + out := make([]string, 0, len(in)) + for m := range in { + out = append(out, m) + } + sort.Strings(out) + return out +} + +// edgeChangeOrder is the edge kind a change order draws. +const edgeChangeOrder = "change-order" + +// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR +// 0235): every module's policy and where it comes from; one module's; or a person's choice for one. +func upgradeCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("upgrade", flag.ContinueOnError) + together := set.Bool("together", false, + "send every machine running it at once, instead of one machine first") + why := set.String("why", "", "why this choice — required for record; kept and said with the policy") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + if len(positionals) == 0 { + all, err := inv.Upgrades(ctx) + if err != nil { + return err + } + names := make([]string, 0, len(all)) + for n := range all { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + u := all[n] + line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From) + if u.Why != "" { + line += ": " + u.Why + } + fmt.Println(line) + } + return nil + } + module := positionals[0] + if len(positionals) == 1 { + decision, err := inv.UpgradeOf(ctx, module) + if err != nil { + return err + } + fmt.Println(sayUpgrade(module, decision)) + return nil + } + + decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()} + switch positionals[1] { + case "roll-out", "roll": + decision.RollOut, decision.Together = true, *together + case "record": + if *together { + // Refused rather than ignored: --together only means anything for a roll-out, and + // accepting it here would store a preference that never applies and looks like it does. + return errors.New("`--together` says how to roll out, so it cannot be given with " + + "`record`, which is the choice not to") + } + if decision.Why == "" { + return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why "+ + "(novox/hq ADR 0236). Nothing was changed", module) + } + case "default": + if err := inv.ClearUpgradeOf(ctx, module); err != nil { + return err + } + decision, err := inv.UpgradeOf(ctx, module) + if err != nil { + return err + } + fmt.Println(sayUpgrade(module, decision)) + // Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14). + rejudgeModule(ctx, open, module) + return nil + default: + return fmt.Errorf("upgrade roll-out|record|default — not %q", positionals[1]) + } + if err := inv.SetUpgradeOf(ctx, module, decision); err != nil { + return err + } + decision, err = inv.UpgradeOf(ctx, module) + if err != nil { + return err + } + fmt.Println(sayUpgrade(module, decision)) + // Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14). + rejudgeModule(ctx, open, module) + return nil +} + +func sayUpgrade(module string, u inventory.Upgrade) string { + from := " (" + u.From + if u.By != "" { + from += ", " + u.By + } + if u.Why != "" { + from += ": " + u.Why + } + from += ")" + if !u.RollOut { + return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+ + "reported as behind until a person pushes them%s", module, from) + } + if u.Together { + return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+ + "together%s", module, from) + } + return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+ + "the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from) } diff --git a/cmd/mesh-controller/urgent_unhealthy_test.go b/cmd/mesh-controller/urgent_unhealthy_test.go deleted file mode 100644 index a1be7af0..00000000 --- a/cmd/mesh-controller/urgent_unhealthy_test.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A module may declare its unhealthy urgent (novox/hq ADR 0355): its module-unhealthy condition is then raised -// urgent; without the field it stays a warning; a wait for a new login stays a warning whatever the module says. - -// judgedTwice judges one machine's statement twice, over a catalogue in which mail declares its unhealthy -// urgent and notes says nothing, and returns what is open. -func judgedTwice(t *testing.T, unhealthy map[string][]inventory.ResourceHealth) []conditions.Condition { - t.Helper() - ctx := t.Context() - k, _ := withConditionsInMemory(t) - was := readHoldingFor - t.Cleanup(func() { readHoldingFor = was }) - readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { - return &holding{ctx: context.Background(), open: open, providers: map[string]providerLookup{}, - shelf: map[string]catalogue.Manifest{ - "mail": {Module: "mail", Version: "1", Unhealthy: &catalogue.UnhealthyPolicy{ - Severity: catalogue.UnhealthyUrgent, Why: "mail is refused or stuck for everyone with a mailbox"}}, - "notes": {Module: "notes", Version: "1"}, - }}, nil - } - streaks := map[string]int{} - for look := 1; look <= 2; look++ { - for m := range unhealthy { - streaks[m] = look - } - if err := judgeModuleHealth(ctx, nil, k, "anchor", unhealthy, streaks, time.Now()); err != nil { - t.Fatal(err) - } - } - open, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return open -} - -func containerDown(module string) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateUnhealthy, Reason: "down"} -} - -func TestAModuleThatDeclaresItsUnhealthyUrgentIsRaisedUrgent(t *testing.T) { - open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {containerDown("mail")}}) - c := conditionOf(open, "module.mail.anchor.unhealthy") - if c == nil { - t.Fatalf("mail's unhealthy condition was not raised: %+v", open) - } - if c.Severity != conditions.Urgent { - t.Errorf("mail declares its unhealthy urgent, and it was raised %s", c.Severity) - } -} - -func TestAModuleThatSaysNothingOfItsUnhealthyStaysAWarning(t *testing.T) { - open := judgedTwice(t, map[string][]inventory.ResourceHealth{"notes": {containerDown("notes")}}) - c := conditionOf(open, "module.notes.anchor.unhealthy") - if c == nil { - t.Fatalf("notes' unhealthy condition was not raised: %+v", open) - } - if c.Severity != conditions.Warning { - t.Errorf("notes says nothing of its unhealthy, and it was raised %s", c.Severity) - } -} - -// Only the module-unhealthy condition changes (ADR 0355 §4): a wait for a new login (ADR 0254) of a module that -// declares its unhealthy urgent stays a warning. -func TestAWaitForANewLoginStaysAWarningWhenTheModuleDeclaresItsUnhealthyUrgent(t *testing.T) { - open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {relogin("mail", "operator"), - userUnit("mail", "operator")}}) - c := conditionOf(open, "module.mail.anchor.relogin-needed") - if c == nil { - t.Fatalf("mail's relogin-needed condition was not raised: %+v", open) - } - if c.Severity != conditions.Warning { - t.Errorf("a wait for a new login was raised %s; want a warning", c.Severity) - } - if conditionOf(open, "module.mail.anchor.unhealthy") != nil { - t.Errorf("a wait for a new login raised mail's unhealthy condition beside it: %+v", open) - } -} diff --git a/cmd/mesh-controller/verb_looks_test.go b/cmd/mesh-controller/verb_looks_test.go deleted file mode 100644 index ced2f5c3..00000000 --- a/cmd/mesh-controller/verb_looks_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "sync/atomic" - "testing" - "time" -) - -// A plan moved by a seat verb is looked at as the verb ends (issue 974, review 1): the verb runs as a child process -// whose saves the serving controller does not hear, so `deliver`, which lets a walk go after the operator's approval, -// would leave its first tier waiting for an unrelated event. -func TestAVerbThatMayMoveAPlanHasTheServingControllerLookWhenItEnds(t *testing.T) { - var looks, failures atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - - was := runTheVerb - runTheVerb = func(context.Context, []string) (verbAnswer, error) { return verbAnswer{}, nil } - t.Cleanup(func() { runTheVerb = was }) - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - for _, c := range []struct { - verb string - args string - look bool - }{ - {"deliver", `{"plan":"p"}`, true}, - {"plans", `{"stop":"plan-1","why":"w"}`, true}, - {"plans", `{}`, false}, - {"nodes", `{}`, false}, - } { - before := looks.Load() - h := handlers[c.verb] - if h == nil { - t.Fatalf("no verb %s", c.verb) - } - if _, err := h(ctx, []byte(c.args)); err != nil { - t.Fatalf("%s %s: %v", c.verb, c.args, err) - } - time.Sleep(100 * time.Millisecond) - if got := looks.Load() > before; got != c.look { - t.Errorf("%s %s: look asked = %v; want %v", c.verb, c.args, got, c.look) - } - } -} diff --git a/cmd/mesh-controller/waiting_provider_test.go b/cmd/mesh-controller/waiting_provider_test.go deleted file mode 100644 index 684921c2..00000000 --- a/cmd/mesh-controller/waiting_provider_test.go +++ /dev/null @@ -1,386 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq -// issue 405, found in the review of ADR 0283's controller change). -// -// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of -// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session -// began before it was, and its unit failed in that account's own service manager. - -// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's -// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait -// names the part that waits by the provision it gives. -func waitingDatabase() inventory.ResourceHealth { - return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence", - What: "the licence key of the database"}) -} - -func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service", - State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits} -} - -// backupsWait is a wait of the same provider for another part than the one its consumers need. -var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"} - -// failingConsumer is a consumer whose check that needs the database fails. -func failingConsumer(module string) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} -} - -// dbSecrets is what the database's waits name: own secrets issued outside the mesh, so a wait for one checks out -// while nobody gave it (ADR 0283 decision 3, novox/hq issue 450). -var dbSecrets = catalogue.OwnSecrets{ - "licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside}, - "backup-key": {Path: "/s/backup-key", IssuedBy: catalogue.IssuedOutside}, -} - -// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the -// catalogue, what was given on the provider's machine (nothing), and each consumer's provider already looked up, -// as providerFor memoises it. -func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding { - h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{}, - shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1", OwnSecrets: dbSecrets, - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}}, - waits: operatorWaitFacts{given: map[string]map[string]time.Time{"db@anchor": {}}}} - for k, p := range bound { - h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true} - } - return h -} - -var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"} - -var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase} - -func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth { - return map[string]inventory.NodeHealth{ - "anchor": {Node: "anchor", Resources: provider}, - "laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}}, - } -} - -// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the -// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is -// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong). -func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) { - shop := []inventory.ResourceHealth{failingConsumer("shop")} - unhealthyDB := waitingDatabase() - unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused" - healthyDB := waitingDatabase() - healthyDB.State, healthyDB.Waits = link.StateHealthy, nil - byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server", - Secret: "licence", What: "the licence key"})), shopOnTheDatabase) - byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", OwnSecrets: dbSecrets, Provides: []catalogue.Offer{{ - Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}} - for _, c := range []struct { - name string - hold *holding - held bool - onlyWaits bool - uncovered bool - }{ - {"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false}, - {"the wait is for the provision's shared credential", byCredential, true, true, false}, - {"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true}, - {"only the needs-operator condition, its parts not said", holdingOf( - []conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}}, - shopFailingBeside(), shopOnTheDatabase), false, false, true}, - {"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false}, - {"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false}, - } { - by, held := c.hold.heldWith("laptop", "shop", shop) - if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits { - t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider, - by.waits, c.held, c.onlyWaits) - } - if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered { - t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered) - } - } -} - -// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a -// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before. -func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) { - now := time.Now() - since := now.Add(-time.Minute) - for _, c := range []struct { - name string - provider inventory.ResourceHealth - want health - says []string - }{ - {"a provider that only waits", waitingDatabase(), healthPerson, - []string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}}, - {"an unhealthy provider", func() inventory.ResourceHealth { - r := waitingDatabase() - r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused" - return r - }(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}}, - } { - healths := shopFailingBeside(c.provider) - for m, h := range healths { - h.HeardAt = now - healths[m] = h - } - f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))} - h, why := moduleHealthWord("shop", "laptop", since, f) - if h != c.want { - t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want) - continue - } - for _, s := range c.says { - if !strings.Contains(why, s) { - t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s) - } - } - } -} - -// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes -// as the statements do. -func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth, - byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition { - t.Helper() - ctx := t.Context() - was := readHoldingFor - t.Cleanup(func() { readHoldingFor = was }) - healths := byProvider - readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { - return holdingOf(open, healths, shopOnTheDatabase), nil - } - for look := 1; look <= 2; look++ { - healths = byProvider - if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider}, - map[string]int{"db": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - for look := 1; look <= 2; look++ { - healths = byConsumer - if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}, - map[string]int{"shop": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - open, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return open -} - -func conditionOf(open []conditions.Condition, key string) *conditions.Condition { - for i, c := range open { - if c.Key == key { - return &open[i] - } - } - return nil -} - -// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and -// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's, -// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved. -func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) { - k, _ := withConditionsInMemory(t) - open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()}, - map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase())) - provider := conditionOf(open, needsOperatorKey("db", "anchor")) - if len(open) != 1 || provider == nil { - t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+ - "provider's needs-operator alone", openKeysOf(open)) - } - if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") { - t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said) - } - if provider.Severity != conditions.Warning { - t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity) - } - if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") { - t.Fatalf("the provider's condition: %+v", provider) - } -} - -// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits. -func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) { - k, _ := withConditionsInMemory(t) - backups := waitingDatabaseFor(backupsWait) - open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups)) - consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop")) - if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil { - t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open)) - } - if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") { - t.Fatalf("the consumer's condition does not say its provider waits: %s", said) - } -} - -// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its -// wait for the password. -func reloginBesideAWait() []inventory.ResourceHealth { - return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"), - waitingResource(passwordWait)} -} - -// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the -// operator owes it. -func TestAWaitBesideAReloginIsSaid(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()}, - map[string]int{"mounts": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open)) - } - if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning { - t.Fatalf("the needs-operator beside the relogin: %+v", got) - } - relogged := false - for _, c := range open { - relogged = relogged || c.Key == reloginKey("mounts", "workstation") - } - if !relogged { - t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open)) - } - // The login done, the wait stays said and the relogin clears. - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}, - map[string]int{"mounts": 3}, time.Now()); err != nil { - t.Fatal(err) - } - got, open = needsOperatorOpen(t, k) - if got == nil || len(open) != 1 { - t.Fatalf("after the new login: %v", openKeysOf(open)) - } -} - -// (2) The same beside a directory used as found. -func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - found := foundDirectory("mounts", time.Now().Add(-time.Hour)) - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ - "mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open)) - } - asFound := false - for _, c := range open { - asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation") - } - if !asFound { - t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open)) - } -} - -// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among -// what fails. -func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process", - Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"} - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ - "mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open)) - } - var fault *conditions.Condition - for i, c := range open { - if c.Key == moduleUnhealthyKey("mounts", "workstation") { - fault = &open[i] - } - } - if fault == nil { - t.Fatalf("the fault is not said: %v", openKeysOf(open)) - } - if strings.Contains(fault.Summary, "mounts.watch") { - t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary) - } -} - -func openKeysOf(cs []conditions.Condition) []string { - var out []string - for _, c := range cs { - out = append(out, c.Key) - } - return out -} - -// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying -// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review). -func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - start := time.Now().Add(-time.Second) - healthy := waitingDatabase() - healthy.State, healthy.Waits = link.StateHealthy, nil - healths := shopFailingBeside(healthy) - was := readHoldingFor - t.Cleanup(func() { readHoldingFor = was }) - readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { - return holdingOf(open, healths, shopOnTheDatabase), nil - } - shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}} - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil { - t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open)) - } - healths = shopFailingBeside(waitingDatabase()) - if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil { - t.Fatal(err) - } - var why string - for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); { - events, err := k.HistorySince(ctx, start) - if err != nil { - t.Fatal(err) - } - for _, e := range events { - if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared { - why = e.Why - } - } - if why == "" { - time.Sleep(10 * time.Millisecond) - } - } - if !strings.Contains(why, "waits on db on anchor, which waits for you") { - t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why) - } -} diff --git a/cmd/mesh-controller/walk_declared.go b/cmd/mesh-controller/walk_declared.go deleted file mode 100644 index 528ebc47..00000000 --- a/cmd/mesh-controller/walk_declared.go +++ /dev/null @@ -1,163 +0,0 @@ -package main - -// The machines whose declaration a walk's merges change (novox/hq ADR 0505 §2): beside what the walk sends, a merge -// can change what another machine is to run — an assignment, a provision or a requirement added or removed — and a -// problem raised there as the walk runs, or as it settles, is the walk's as much as one on a machine it sends to. The -// router holds what a walk touches (ADR 0308 §1); these are touches of their own, marked `declared`. -// -// **Compared as the mesh compares whether a machine is up to date.** Each machine's declaration is composed as a push -// would compose it, with the number, epoch and generation it was last sent, and its digest held against the one it was -// last sent (wouldSend, inventory.Waiting): byte for byte what it was sent when nothing changed, so a secret, a sequence -// or a lease's epoch never makes a machine differ by itself. A machine whose declaration cannot be worked out now and -// was sent one before is taken as changed: not known is changed, never "unchanged" (a merge that leaves a machine -// unable to compose is the case this exists for, the 11:03 message of ADR 0505's context). -// -// **Said again, not waited for.** Composing every machine takes seconds; the walk's start and end are said at once as -// before, and the serving controller, which hears every `plan-moved` — its own and a verb's — says a start or an end -// again with the declared touches added, marked `declared_read`, which it never reads twice. The router merges a walk -// said again (messenger walks.go). - -import ( - "context" - "encoding/json" - "fmt" - "os" - "sort" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// walkDeclaredRead marks a walk's start or end said again with its declared touches: read once. -const walkDeclaredRead = "declared_read" - -// declaredWithin bounds the comparison of every machine for one walk's start or end. -const declaredWithin = time.Minute - -// changedDeclarations is every machine whose declaration differs from what it was last sent, and every machine sent -// one before whose declaration cannot be worked out now. A seam a test replaces. -var changedDeclarations = func(ctx context.Context, open *stores) ([]string, error) { - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return nil, err - } - ctx = withInputCache(ctx) - would, err := wouldSend(ctx, open, nodes) - if err != nil { - return nil, err - } - behind, err := open.inventory.Waiting(ctx, would) - if err != nil { - return nil, err - } - sent := map[string]bool{} - for _, n := range nodes { - if s, known, err := open.inventory.SentTo(ctx, n.Name); err == nil && known && s.Digest != "" { - sent[n.Name] = true - } - } - return changedOf(nodes, would, behind, sent), nil -} - -// changedOf is changedDeclarations over what was read: the machines behind, and those sent one before that could not -// be worked out. Pure; sorted. -func changedOf(nodes []inventory.Node, would map[string]string, behind []inventory.Machine, sent map[string]bool) []string { - set := map[string]bool{} - for _, m := range behind { - set[m.Node] = true - } - for _, n := range nodes { - if _, worked := would[n.Name]; !worked && sent[n.Name] { - set[n.Name] = true - } - } - out := make([]string, 0, len(set)) - for n := range set { - out = append(out, n) - } - sort.Strings(out) - return out -} - -// withDeclared is a walk's start or end said again with the machines whose declaration changed added as declared -// touches: false when the body says no start or end, was said again already, or names no machine it does not touch. -func withDeclared(body []byte, changed []string) ([]byte, bool, error) { - said, wanted, err := toSayAgain(body) - if err != nil || !wanted { - return nil, false, err - } - var touches []walkTouch - if raw := said["touches"]; raw != nil && string(raw) != "null" { - if err := json.Unmarshal(raw, &touches); err != nil { - return nil, false, fmt.Errorf("its touches are not a list of {machine, modules}: %w", err) - } - } - added := declaredTouches(touches, changed) - if len(added) == 0 { - return nil, false, nil - } - touches = append(touches, added...) - sort.Slice(touches, func(i, j int) bool { return touches[i].Machine < touches[j].Machine }) - if said["touches"], err = json.Marshal(touches); err != nil { - return nil, false, err - } - said[walkDeclaredRead] = json.RawMessage(`true`) - out, err := json.Marshal(said) - return out, err == nil, err -} - -// toSayAgain reads a plan-moved: whether it says a walk's start or end not said again yet. -func toSayAgain(body []byte) (map[string]json.RawMessage, bool, error) { - var said map[string]json.RawMessage - if err := json.Unmarshal(body, &said); err != nil { - return nil, false, fmt.Errorf("a plan-moved that is not an object: %w", err) - } - var phase string - _ = json.Unmarshal(said["phase"], &phase) - return said, (phase == inventory.WalkStarted || phase == inventory.WalkEnded) && said[walkDeclaredRead] == nil, nil -} - -// declaredTouches are the machines changed that a walk's touches do not name, each a declared touch. Pure; sorted. -func declaredTouches(touches []walkTouch, changed []string) []walkTouch { - named := map[string]bool{} - for _, t := range touches { - named[t.Machine] = true - } - var out []walkTouch - for _, n := range changed { - if !named[n] { - out = append(out, walkTouch{Machine: n, Modules: []string{}, Declared: true}) - named[n] = true - } - } - sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine }) - return out -} - -// sayDeclared says a walk's start or end again with its declared touches, when it has any. What cannot be read is -// said on the controller's log: the walk is then held as before, never less. -func sayDeclared(ctx context.Context, bus link.Bus, open *stores, body []byte) { - if _, wanted, _ := toSayAgain(body); !wanted { - return // no start or end, or said again already: nothing to compare - } - comparing, cancel := context.WithTimeout(ctx, declaredWithin) - defer cancel() - changed, err := changedDeclarations(comparing, open) - if err != nil { - fmt.Fprintf(os.Stderr, "plan-moved: which machines' declarations the walk changes cannot be read: %v\n", err) - return - } - again, ok, err := withDeclared(body, changed) - if err != nil || !ok { - if err != nil { - fmt.Fprintf(os.Stderr, "plan-moved: the walk cannot be said again with its declared machines: %v\n", err) - } - return - } - stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer stop() - if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, again); err != nil { - fmt.Fprintf(os.Stderr, "plan-moved: the walk could not be said again with its declared machines: %v\n", err) - } -} diff --git a/cmd/mesh-controller/walk_declared_test.go b/cmd/mesh-controller/walk_declared_test.go deleted file mode 100644 index 2dd47889..00000000 --- a/cmd/mesh-controller/walk_declared_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0505 §2: the machines whose declaration a walk's merges change are said as declared touches. - -// The machines behind what they were last sent, and those sent one before that cannot be worked out now, are changed; -// one never sent and not worked out is not. -func TestAMachineBehindOrNoLongerComposableIsChanged(t *testing.T) { - nodes := []inventory.Node{{Name: "ace"}, {Name: "g14"}, {Name: "novox"}, {Name: "shanks"}, {Name: "new"}} - would := map[string]string{"ace": "d1", "g14": "d2", "novox": "d3"} - behind := []inventory.Machine{{Node: "ace"}} - sent := map[string]bool{"ace": true, "g14": true, "novox": true, "shanks": true} - if got := changedOf(nodes, would, behind, sent); !reflect.DeepEqual(got, []string{"ace", "shanks"}) { - t.Fatalf("changed: %v, want ace (behind) and shanks (no longer composes)", got) - } -} - -// A walk's start said by plan-moved (walk_said.go's body), said again with the changed machines it does not touch as -// declared touches, once. -func TestAWalksStartIsSaidAgainWithTheMachinesItsMergesChange(t *testing.T) { - body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding}, - walkSaid: &walkSaid{Walk: "plan-1", Phase: inventory.WalkStarted, BudgetClass: "leaf", BoundSeconds: 600, - Touches: []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}}}}) - if err != nil { - t.Fatal(err) - } - again, ok, err := withDeclared(body, []string{"novox", "shanks"}) - if err != nil || !ok { - t.Fatalf("not said again: %v %v", ok, err) - } - var said struct { - ID string `json:"id"` - Phase string `json:"phase"` - Touches []walkTouch `json:"touches"` - DeclaredRead bool `json:"declared_read"` - } - if err := json.Unmarshal(again, &said); err != nil { - t.Fatal(err) - } - want := []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}, {Machine: "shanks", Modules: []string{}, Declared: true}} - if said.ID != "plan-1" || said.Phase != inventory.WalkStarted || !said.DeclaredRead || !reflect.DeepEqual(said.Touches, want) { - t.Fatalf("said again: %+v", said) - } - if _, ok, _ := withDeclared(again, []string{"ace"}); ok { - t.Fatalf("a start said again was said a third time") - } - if _, ok, _ := withDeclared(body, []string{"novox"}); ok { - t.Fatalf("said again with nothing it does not touch") - } - moved, _ := json.Marshal(inventory.Plan{ID: "plan-2", State: inventory.PlanBuilding}) - if _, ok, _ := withDeclared(moved, []string{"shanks"}); ok { - t.Fatalf("a move inside a walk was said as its start") - } -} diff --git a/cmd/mesh-controller/walk_from_keys_integration_test.go b/cmd/mesh-controller/walk_from_keys_integration_test.go deleted file mode 100644 index a63213ab..00000000 --- a/cmd/mesh-controller/walk_from_keys_integration_test.go +++ /dev/null @@ -1,186 +0,0 @@ -//go:build integration - -package main - -import ( - "github.com/nats-io/nats.go" - "reflect" - "strings" - "testing" - "time" - - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A6, A7 whole against the store and a real bus (ADR 0503 §6, ADR 0498 check §7; issue 1851): a walk whose modules the -// builder built goes from asked to built with no build asked, its keys taken in off the plans' hold, then sends its tier -// to the first machine as before; a key still building holds the tier, said, and moves on when the builder's key is done. -func TestAWalkTakesItsTierFromTheBuildersKeysAndSendsIt(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "app1", "app2") - ctx := t.Context() - inv := tm.open.inventory - noBuildAsked(t) - b := aBuilderOnTheBus(t) - - // The walk carries c3 of the catalogue; its tier is unasked. - p := tm.plan(t) - p.Commit = "c3c3c3c3c3c3c3c3" - for _, m := range []string{"app1", "app2"} { - p.Modules[m] = &inventory.PlanModule{} - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - fp1, fp2 := fpOf('1'), fpOf('2') - m1 := b.indexed(t, "app1", "c3", "novox/mesh-catalog", "modules/app1", p.Commit, fp1) - m2 := b.indexed(t, "app2", "c3", "novox/mesh-catalog", "modules/app2", p.Commit, fp2) - b.built(t, "app1", "c3", fp1, m1, sdkdelivery.BuildDone) - b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildBuilding) - - advancePlans(ctx, tm.open) - becomes(t, 10*time.Second, "app1 taken from its key", func() bool { - advancePlans(ctx, tm.open) - return tm.plan(t).Modules["app1"].State == "built" - }) - p = tm.plan(t) - if s := p.Modules["app2"]; s.State != "asked" || !s.Key || !strings.Contains(s.Why, "building on ace") { - t.Fatalf("A7: app2, whose key is building, is %+v", s) - } - if len(tm.sent) != 0 { - t.Fatalf("A7: the tier was sent while app2 waits on its key: %v", tm.sent) - } - if s := p.Modules["app1"]; !strings.Contains(s.Made, "taken from the builder's key") || s.Fingerprint != fp1 { - t.Fatalf("A6: plans does not say app1 was taken from the builder's key: %+v", s) - } - - b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildDone) - becomes(t, 10*time.Second, "the tier sent to the first machine", func() bool { - advancePlans(ctx, tm.open) - return len(tm.sent) > 0 - }) - p = tm.plan(t) - if !reflect.DeepEqual(tm.sent[0], []string{"anchor"}) { - t.Fatalf("A6: the tier taken from keys was sent %v, not to the first machine", tm.sent) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - for _, m := range []string{"app1", "app2"} { - if current[m].Commit != p.Commit || !strings.HasPrefix(p.Modules[m].Build, "key-") { - t.Fatalf("A6: %s stands on %+v (%s), not the key's build at %s", m, current[m], p.Modules[m].Build, p.Commit) - } - } -} - -// A7: a failed key fails the walk with the builder's words, and nothing is sent. -func TestAFailedKeyFailsTheWalkInTheBuildersWords(t *testing.T) { - tm := aTierMesh(t, "app1") - ctx := t.Context() - inv := tm.open.inventory - noBuildAsked(t) - b := aBuilderOnTheBus(t) - p := tm.plan(t) - p.Commit = "c4c4c4c4c4c4c4c4" - p.Modules["app1"] = &inventory.PlanModule{} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - fp := fpOf('4') - manifest := b.indexed(t, "app1", "c4", "novox/mesh-catalog", "modules/app1", p.Commit, fp) - b.built(t, "app1", "c4", fp, manifest, sdkdelivery.BuildFailed) - for i := 0; i < 3; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "digests differ: ace sha256:1, g14 sha256:2") { - t.Fatalf("A7: the walk is %s: %q", p.State, p.Note) - } - if len(tm.sent) != 0 { - t.Fatalf("A7: a failed walk sent %v", tm.sent) - } -} - -// C2's criterion (issue 1851, R4g): a merge whose walk moves the builder, a build module and the controller takes every one -// from the builder's referenced done key, and no ask reaches any subject of the seat node-build-agent. A fake builder writes -// the keys on a real bus. -func TestTheBuilderABuildModuleAndTheControllerAreTakenFromKeysAndNothingIsAskedOfBuildAgent(t *testing.T) { - names := []string{sdkdelivery.BuilderModule, "mesh-build-go", "mesh-controller"} - tm := aTierMesh(t, names...) - ctx := t.Context() - inv := tm.open.inventory - b := aBuilderOnTheBus(t) - asked := make(chan string, 16) - sub, err := b.js.Conn().Subscribe("mesh.seat.node-build-agent.>", func(m *nats.Msg) { asked <- m.Subject }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - p := tm.plan(t) - p.Commit = "c5c5c5c5c5c5c5c5" - for _, m := range names { - p.Modules[m] = &inventory.PlanModule{} - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - for i, m := range names { - fp := fpOf(byte('a' + i)) - manifest := b.indexed(t, m, "c5", "novox/mesh-catalog", "modules/"+m, p.Commit, fp) - b.built(t, m, "c5", fp, manifest, sdkdelivery.BuildDone) - } - becomes(t, 10*time.Second, "the three taken from their keys", func() bool { - advancePlans(ctx, tm.open) - q := tm.plan(t) - for _, m := range names { - if s := q.Modules[m]; s == nil || s.State != "built" { - return false - } - } - return true - }) - p = tm.plan(t) - for _, m := range names { - if s := p.Modules[m]; !s.Key || !strings.HasPrefix(s.Build, "key-") || !strings.Contains(s.Made, "taken from the builder's key") { - t.Errorf("%s was not taken from the builder's referenced key: %+v", m, s) - } - } - select { - case subject := <-asked: - t.Fatalf("an ask reached the retired build seat: %s", subject) - case <-time.After(300 * time.Millisecond): - } -} - -// C5's criterion: after a plan that moves the builder, its record names the builder's key and its fingerprint. -func TestTheBuildersRecordNamesItsKeyAndFingerprint(t *testing.T) { - tm := aTierMesh(t, sdkdelivery.BuilderModule) - ctx := t.Context() - inv := tm.open.inventory - b := aBuilderOnTheBus(t) - p := tm.plan(t) - p.Commit = "c6c6c6c6c6c6c6c6" - p.Modules[sdkdelivery.BuilderModule] = &inventory.PlanModule{} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - fp := fpOf('9') - manifest := b.indexed(t, sdkdelivery.BuilderModule, "c6", "novox/mesh-catalog", "modules/"+sdkdelivery.BuilderModule, p.Commit, fp) - b.built(t, sdkdelivery.BuilderModule, "c6", fp, manifest, sdkdelivery.BuildDone) - becomes(t, 10*time.Second, "the builder taken from its key", func() bool { - advancePlans(ctx, tm.open) - return tm.plan(t).Modules[sdkdelivery.BuilderModule].State == "built" - }) - id := tm.plan(t).Modules[sdkdelivery.BuilderModule].Build - record, found, err := inv.BuildByID(ctx, id) - if err != nil || !found { - t.Fatalf("the builder's record %q: %v %v", id, found, err) - } - if !strings.HasPrefix(record.ID, "key-") || record.SourceFingerprint != fp || record.Module != sdkdelivery.BuilderModule { - t.Fatalf("the builder's record names %+v, not its key and fingerprint %s", record, fp) - } -} diff --git a/cmd/mesh-controller/walk_keys_scaffold_integration_test.go b/cmd/mesh-controller/walk_keys_scaffold_integration_test.go deleted file mode 100644 index d957e9c0..00000000 --- a/cmd/mesh-controller/walk_keys_scaffold_integration_test.go +++ /dev/null @@ -1,137 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk takes every module from the builder's done key and asks no build (R4g, ADR 0501, ADR 0503 §6). The tests of what -// a walk does around its builds (its tiers, sends, stops, retries, backups and waits for its delivery's word) need the -// builds to happen without a builder, so this stands one in: every module has a done key at once, and the take-in of a key -// records the build and registers the module as the real one does, in the caller's turn. The tests that are of the take-in -// itself (key_takein, walk_from_keys, first_builds) use a builder on a bus of their own (builder_on_the_bus_test.go). - -// keysAllDone is the indexer and the builder as a walk reads them when every module is built at every commit. -type keysAllDone struct { - mu sync.Mutex - paths [][3]string // repository, path and commit of each key taken in, in order - named []string // the same, as repository#id -} - -// fpOfModuleAt is the fingerprint a module has at a commit in these tests: one for each pair. -func fpOfModuleAt(module, commit string) string { - sum := sha256.Sum256([]byte(module + "@" + commit)) - return "src2:" + hex.EncodeToString(sum[:]) -} - -func (k *keysAllDone) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { - return sdkdelivery.Module{FP2: fpOfModuleAt(module, commit), Commit: commit, Repository: "novox/mesh-catalog", - Path: "modules/" + module}, nil -} - -func (k *keysAllDone) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { - digest := "sha256:" + fp[len("src2:"):] - return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Artifacts: []sdkdelivery.BuiltArtifact{{ - Name: "a", Kind: "archive", Digest: digest, Reference: "artifact-store://" + module + "/a/blobs/" + digest}}}}, nil -} - -// take is the take-in of a key as the walk's look starts it: the build recorded and the module registered at the commit the -// walk carries. -func (k *keysAllDone) take(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) { - inv := open.inventory - defer func() { - keyTakes.Lock() - delete(keyTakes.running, id) - keyTakes.taken[id] = true - keyTakes.Unlock() - askToLook() - }() - m := catalogue.Manifest{Module: module, Version: "1"} - if shelf, err := inv.Catalogue(ctx); err == nil { - if held, ok := shelf[module]; ok { - m = held - } - } - src, err := inv.SourceOf(ctx, module) - if err != nil { - src = inventory.Source{Repository: trunk.Repository, Seat: "git", Ref: "main", Path: keyPath(trunk)} - } - src.BuiltFrom, src.Head = trunk.Commit, trunk.Commit - raw, _ := json.Marshal(m) - if err := inv.RecordBuild(ctx, inventory.Build{ID: id, Module: module, Repository: src.Repository, Commit: trunk.Commit, - Path: src.Path, On: "ace", At: time.Now().UTC(), Manifest: raw, SourceFingerprint: trunk.FP2}); err != nil { - keepKeyWait(id, "the build could not be recorded: "+err.Error()) - return - } - if err := inv.RegisterModule(ctx, m, src); err != nil { - keepKeyWait(id, "the module could not be registered: "+err.Error()) - return - } - k.mu.Lock() - defer k.mu.Unlock() - k.paths = append(k.paths, [3]string{src.Repository, src.Path, trunk.Commit}) - k.named = append(k.named, src.Repository+"#"+id) -} - -// everyKeyDone has the walks of a test take every module from a done key, and says which were taken. -func everyKeyDone(t *testing.T) *keysAllDone { - t.Helper() - k := &keysAllDone{} - reset := func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - } - reset() - wasKeys, wasTake, wasBackground := openPlanKeys, takeKeyIn, inBackground - openPlanKeys = func() (planKeyReads, func(), error) { return k, func() {}, nil } - takeKeyIn = k.take - inBackground = func(work func()) { work() } - t.Cleanup(func() { - openPlanKeys, takeKeyIn, inBackground = wasKeys, wasTake, wasBackground - reset() - }) - return k -} - -// asksRecorded has the walks of a test take every module from a done key, and says each as repository#build id. -func asksRecorded(t *testing.T) *[]string { - t.Helper() - return &everyKeyDone(t).named -} - -// asksWithPaths is asksRecorded saying each as repository, path and the commit the walk carried. -func asksWithPaths(t *testing.T) *[][3]string { - t.Helper() - return &everyKeyDone(t).paths -} - -// asksOfEveryModule once made every module the builder's own, the one ask left; a walk asks nothing now. -func asksOfEveryModule(t *testing.T) { t.Helper() } - -// noBuildAsked once failed a test that asked build-agent for a build; nothing can now, so it says nothing. -func noBuildAsked(t *testing.T) { t.Helper() } - -// twoTiers registers two modules, b standing on a, each with a source a plan asks. -func twoTiers(t *testing.T, open *stores) { - t.Helper() - for _, name := range []string{"a", "b"} { - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"}, - inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil { - t.Fatal(err) - } - } -} diff --git a/cmd/mesh-controller/walk_resumed_silence_1886_test.go b/cmd/mesh-controller/walk_resumed_silence_1886_test.go deleted file mode 100644 index bb566c7f..00000000 --- a/cmd/mesh-controller/walk_resumed_silence_1886_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package main - -import ( - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// Issue 1886, the resume of R4e (plan-1792707174627079192): tier 0 was sent at 00:23:28, the walk was stopped at -// 02:01 and resumed at 07:43, and tier 1 was sent at 07:49:54. 35 seconds later the gate named ace silent "within -// 15m0s", and at 07:51:30 shanks, in the same second shanks reported it applied: the plan failed as lost. The walk -// judged each machine on every send it had made it, tier 0's of 00:23 too, narrowed to every module the walk carried -// there — so tier 0's send was held against the templates of tier 1's modules it never carried, and silence was -// measured from 00:23. A machine is judged on the walk's last send to it: that send carries its whole declaration. - -func r4eResumed() (inventory.Plan, []inventory.Reported, inventory.ProvLookup, time.Time) { - tier0 := time.Date(2026, 10, 22, 22, 23, 28, 0, time.UTC) - tier1 := time.Date(2026, 10, 23, 5, 49, 54, 0, time.UTC) - now := time.Date(2026, 10, 23, 5, 51, 30, 0, time.UTC) - t0, t1 := tier0, tier1 - p := inventory.Plan{ID: "plan-1792707174627079192", State: inventory.PlanRolling, Tier: 1, - Tiers: [][]string{{"pacman"}, {"mesh-build-image"}, {"gitea"}}, - Modules: map[string]*inventory.PlanModule{ - "pacman": {State: "built", SentAt: &t0, Rest: map[string]inventory.SentDeclaration{ - "shanks": {Digest: "d0-shanks", Sequence: 900, At: tier0, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}}, - "ace": {Digest: "d0-ace", Sequence: 1100, At: tier0, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}}}}, - "mesh-build-image": {State: "built", SentAt: &t1, Rest: map[string]inventory.SentDeclaration{ - "shanks": {Digest: "d1-shanks", Sequence: 971, At: tier1, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}}, - "ace": {Digest: "d1-ace", Sequence: 1212, At: tier1, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}}}}, - "gitea": {}}} - shanksAt := now - aceAt := time.Date(2026, 10, 23, 0, 57, 50, 0, time.UTC) - reports := []inventory.Reported{ - // shanks reported on tier 1's send in the second the gate looked. - {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &shanksAt, Declared: "d1-shanks", Current: true, - ReportedSequence: 971}, - // ace's last report is from before tier 1's send, on a declaration sequenced after tier 0's. - {Node: "ace", Outcome: inventory.OutcomeApplied, At: &aceAt, Declared: "d-ace-0257", ReportedSequence: 1150}, - } - says := map[string]inventory.ProvReport{ - "shanks/pacman": {Status: inventory.ProvApplied, Templates: "pac-1", At: now}, - "shanks/mesh-build-image": {Status: inventory.ProvApplied, Templates: "img-new", At: now}, - "ace/pacman": {Status: inventory.ProvApplied, Templates: "pac-1", At: aceAt}, - // ace's client had not placed the old image's files (issue 1884), and has not yet the new one's. - "ace/mesh-build-image": {Status: inventory.ProvWaiting, Templates: "img-old", At: aceAt}, - } - prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - return p, reports, prov, now -} - -func TestAResumedWalkJudgesEachMachineOnItsLastSend(t *testing.T) { - p, reports, prov, now := r4eResumed() - waiting, failed, silent := walkReports(p, reports, now, prov) - if len(silent) != 0 { - t.Fatalf("1886: %v named silent within %s of tier 1's send, from tier 0's send of 00:23", silent, applySilentAfter) - } - if len(failed) != 0 || !slices.Equal(waiting, []string{"ace"}) { - t.Fatalf("1886: waiting %v, failed %v; want ace waited for (sent 96 s ago) and shanks reported", waiting, failed) - } - // The gate the plan failed at: mesh-build-image runs on ace and shanks, which tier 2 needs. - needs := []gateNeed{{module: "mesh-build-image", running: []string{"ace", "shanks"}, - since: func(string) time.Time { return *p.Modules["mesh-build-image"].SentAt }}} - w, quiet, lost := settleTierGate(needs, reports, silentNodes(p, reports, now, prov)) - if lost || len(quiet) != 0 || !slices.Equal(w, []string{"mesh-build-image on ace"}) { - t.Fatalf("1886: the gate says waiting %v, silent %v, lost %v; want mesh-build-image on ace waited for", w, quiet, lost) - } -} - -// The machine silent on the walk's last send to it is still silent once the bound has passed since that send. -func TestAMachineSilentOnItsLastSendIsStillSilent(t *testing.T) { - p, reports, prov, now := r4eResumed() - later := now.Add(applySilentAfter + time.Minute) - _, _, silent := walkReports(p, reports, later, prov) - if !slices.Equal(silent, []string{"ace"}) { - t.Fatalf("1886: silent %v past the bound from tier 1's send; want ace", silent) - } -} - -// A walk failed only because every machine its gate waited on was silent, whose tiers are all sent, is carried on by -// `plans retry` once those machines reported (issue 1886): it failed on no build and no rollout, so the retry of failed -// builds and of stopped rollouts refused it with "nothing in tier 1 … failed to build or stopped rolling out". -func r4eLost() inventory.Plan { - p, _, _, _ := r4eResumed() - p.State = inventory.PlanFailed - p.Note = "tier 1 built; " + silentGateWords + " 15m0s: shanks" - // mesh-tools is built and recorded, never sent, as R4e's was. - p.Tiers[1] = append(p.Tiers[1], "mesh-tools") - p.Modules["mesh-tools"] = &inventory.PlanModule{State: "built"} - return p -} - -func TestRetryCarriesOnAWalkLostToSilenceOnceItsMachinesReported(t *testing.T) { - p, reports, prov, now := r4eResumed() - p = r4eLost() - if !lostToSilence(p) { - t.Fatalf("1886: the plan R4e failed as is not read as lost to silence: %q", p.Note) - } - if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil { - t.Fatalf("1886: a walk lost to silence was refused a retry: %v", err) - } - reports[1].Current, reports[1].Declared, reports[1].ReportedSequence = true, "d1-ace", 1212 - reports[1].At = &now - says := map[string]inventory.ProvReport{} - for _, n := range []string{"ace", "shanks"} { - says[n+"/pacman"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "pac-1", At: now} - says[n+"/mesh-build-image"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "img-new", At: now} - } - prov = func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - said, err := resumeLost(&p, silentNodes(p, reports, now.Add(time.Hour), prov)) - if err != nil { - t.Fatalf("1886: the resume of a walk whose machines reported refused: %v", err) - } - if p.State != inventory.PlanBuilding || p.Tier != 1 { - t.Fatalf("1886: resumed walk is %s at tier %d; want building at tier 1, for the next look to pass its gate", p.State, p.Tier) - } - if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(said, "resumed at tier 2 of 3") { - t.Fatalf("1886: note %q, said %q", p.Note, said) - } -} - -// Every machine its tier was sent to still silent: the gate would fail it again, so the retry says so instead. -func TestRetryRefusesAWalkLostToSilenceWhileEveryMachineIsStillSilent(t *testing.T) { - p := r4eLost() - if _, err := resumeLost(&p, map[string]bool{"ace": true, "shanks": true}); err == nil || !strings.Contains(err.Error(), "ace, shanks") { - t.Fatalf("1886: a walk whose machine is still silent was resumed: %v", err) - } - if p.State != inventory.PlanFailed { - t.Fatalf("1886: a refused resume changed the plan to %s", p.State) - } -} - -// A walk lost to silence with something of its tier never sent is not this: its send is the look's to make. -func TestALostWalkWithATierUnsentIsNotLostToSilence(t *testing.T) { - p := r4eLost() - p.Modules["mesh-build-image"].State = "asked" - if lostToSilence(p) { - t.Fatal("1886: a tier with a module still asked was read as lost to silence") - } - p = r4eLost() - p.Note = "tier 1 built; mesh-build-image failed to build" - if lostToSilence(p) { - t.Fatal("1886: a plan failed by a build was read as lost to silence") - } -} - -// A walk lost to silence is refused its resume beside a newer open plan of its repository and branch, which supersedes -// it, and beside a related open plan: one plan at a time (ADR 0531). -func TestRetryRefusesAWalkLostToSilenceBesideANewerOrARelatedOpenPlan(t *testing.T) { - p := r4eLost() - created := time.Date(2026, 10, 22, 22, 15, 9, 0, time.UTC) - p.Repository, p.Branch, p.Commit, p.Created = "novox/mesh-sdk", "main", "7aacbc16", created - newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed", - Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}}, - Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}} - if err := retryRefusal(p, []inventory.Plan{p, newerOpen}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "plan-newer supersedes it") { - t.Errorf("1886: a walk lost to silence was resumed under a newer open plan of its branch: %v", err) - } - beside := inventory.Plan{ID: "plan-beside", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a", - Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-build-image"}}, - Modules: map[string]*inventory.PlanModule{"mesh-build-image": {State: "asked"}}} - if err := retryRefusal(p, []inventory.Plan{p, beside}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "one walk at a time") { - t.Errorf("1886: a walk lost to silence was resumed beside a related open plan: %v", err) - } -} diff --git a/cmd/mesh-controller/walk_said.go b/cmd/mesh-controller/walk_said.go deleted file mode 100644 index d1a8d3b5..00000000 --- a/cmd/mesh-controller/walk_said.go +++ /dev/null @@ -1,343 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "slices" - "sort" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// What a walk touches, its bound and its outcome (novox/hq ADR 0308 decision 1, decision 5): said in -// `plan-moved` on the save that starts the walk and the save that ends it, beside the walk whole, for the holder -// of the operator's channel to hold what the walk touches while it runs and tell its result once. -// -// **What a walk touches** is every module it moves on every machine it sends it to, and every module a moved -// tool runner (node-tools) or node-engine (mesh-host) carries on that machine: what restarts there. At the start -// that is what the walk will send — each module of its tiers whose policy rolls it out, on every machine running -// it; at the end it is what it reached — the machines each module was sent to, first and rest, and what a gated -// send carried with it. -// -// **Its bound** is twice the largest delivery budget among the walk's deliveries (deliveryBudgets: a core -// module's ten minutes, a leaf module's budget in force), and fifteen minutes, not doubled, for a walk none of -// whose deliveries has a budget: a release plan (builds waiting for a person's gate), a walk whose every module's -// policy records rather than rolls out (held for a person, ADR 0282 decision 1), or a walk kept without a class. - -// walkNoBudgetBound is the bound of a walk with no budget (ADR 0308 decision 5): not doubled. -const walkNoBudgetBound = 15 * time.Minute - -// budgetNone is the budget class of a walk none of whose deliveries has a budget. -const budgetNone = "none" - -// walkSaid is what `plan-moved` adds on a walk's start and end, beside the walk whole. -type walkSaid struct { - Walk string `json:"walk"` - Phase string `json:"phase"` - // Touches is, per machine, the modules the walk moves or reaches there and what a moved tool runner or - // node-engine carries there; never null. - Touches []walkTouch `json:"touches"` - // Outcome is on the end only: delivered, failed or stopped. - Outcome string `json:"outcome,omitempty"` - // BoundSeconds and BudgetClass are said on both, read from the walk's class at the start. - BoundSeconds int `json:"bound_seconds"` - BudgetClass string `json:"budget_class"` - // StartedAt is when the walk was said started, on both; EndedAt when it was said ended, on the end: the - // bound runs from StartedAt, whenever the event is heard. - StartedAt *time.Time `json:"started_at,omitempty"` - EndedAt *time.Time `json:"ended_at,omitempty"` -} - -// walkTouch is what a walk touches on one machine. -type walkTouch struct { - Machine string `json:"machine"` - Modules []string `json:"modules"` - // Carried names those of Modules that are there only because a moved tool runner or node-engine carries - // them: what restarts with it, not what the walk builds. - Carried []string `json:"carried,omitempty"` - // Declared marks a machine the walk sends nothing to whose declaration its merges change (novox/hq ADR 0505 §2): - // said again by the serving controller once it has compared every machine (walk_declared.go). - Declared bool `json:"declared,omitempty"` -} - -// planMovedBody is `plan-moved`'s body: the walk whole, as before, and on its start and end what it touches. -type planMovedBody struct { - inventory.Plan - *walkSaid -} - -// walkFacts is what the touches are read from: the controller's inventory. -type walkFacts interface { - Running(ctx context.Context, module string) ([]string, error) - UpgradeOf(ctx context.Context, module string) (inventory.Upgrade, error) - Assigned(ctx context.Context, node string) ([]string, error) - SentBuilds(ctx context.Context, node string) (map[string]string, bool, error) - Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error) -} - -// sayWalk is what `plan-moved` adds to the save that started or ended the walk; nil on every other save. A fact -// that cannot be read is said on the controller's log and left out, never in the way of the event: a walk said -// with fewer touches is held less, never longer. -func sayWalk(ctx context.Context, facts walkFacts, p inventory.Plan) *walkSaid { - if p.WalkSaid == "" { - return nil - } - w := &walkSaid{Walk: p.ID, Phase: p.WalkSaid} - if p.Times != nil { - w.StartedAt = p.Times.Started - if p.WalkSaid == inventory.WalkEnded { - w.EndedAt, w.Outcome = p.Times.Ended, p.Outcome() - } - } - read := readWalkFacts(ctx, facts, p) - w.BudgetClass = walkBudgetClass(p, read.rollsOut) - w.BoundSeconds = int(walkBound(w.BudgetClass) / time.Second) - var moved map[string][]string - if p.WalkSaid == inventory.WalkStarted { - moved = walkSends(p, read.running, read.rollsOut) - } else { - moved = walkReached(p, read.running) - } - var shelf map[string]catalogue.Manifest - if len(moved[broker.RuntimeModule]) > 0 { - // Which modules the tool runner serves is the catalogue's: read only when the walk moves it. - shelf = read.shelf() - } - w.Touches = touchesOf(moved, read.onMachine(moved), shelf) - return w -} - -// walkBudgetClass is the largest budget class among the walk's deliveries: its class read at the cut when one of -// its modules rolls out, and none when it has no budget. -func walkBudgetClass(p inventory.Plan, rollsOut func(string) bool) string { - if p.Release != nil || p.Times == nil { - return budgetNone - } - if _, budgeted := deliveryBudgets[p.Times.Class]; !budgeted { - return budgetNone - } - for _, tier := range p.Tiers { - for _, m := range tier { - if rollsOut(m) { - return p.Times.Class - } - } - } - return budgetNone -} - -// walkBound is a walk's bound by its budget class: twice its budget, or fifteen minutes with none. -func walkBound(class string) time.Duration { - if b, ok := deliveryBudgets[class]; ok { - return 2 * b - } - return walkNoBudgetBound -} - -// walkSends is what a walk will send at its start, per module the machines it goes to: each module of its tiers -// whose policy rolls it out, on every machine running it (for a release plan, those of its order). -func walkSends(p inventory.Plan, running func(string) []string, rollsOut func(string) bool) map[string][]string { - out := map[string][]string{} - for _, tier := range p.Tiers { - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.State == planDeleted { - continue - } - if p.Release == nil && !rollsOut(m) { - continue - } - for _, n := range running(m) { - if p.Release != nil && !slices.Contains(p.Release.Order, n) { - continue - } - out[m] = appendNew(out[m], n) - } - } - } - return out -} - -// walkReached is what a walk sent by its end, per module the machines it reached: its first machines, its rest, -// and every module a gated send carried with it. A release plan reached the machines it walked: those done and -// the one being judged. -func walkReached(p inventory.Plan, running func(string) []string) map[string][]string { - out := map[string][]string{} - carried := func(g *inventory.PlanGate) { - if g == nil { - return - } - for _, c := range g.Carried { - out[c.Module] = appendNew(out[c.Module], c.Node) - } - } - if r := p.Release; r != nil { - reached := append([]string{}, r.Done...) - if r.Gate != nil { - reached = appendNew(reached, r.Gate.Machines...) - carried(r.Gate) - } - for _, g := range r.Gates { - if g != nil && !g.Pending { - reached = appendNew(reached, g.Machines...) - carried(g) - } - } - for _, tier := range p.Tiers { - for _, m := range tier { - for _, n := range running(m) { - if slices.Contains(reached, n) { - out[m] = appendNew(out[m], n) - } - } - } - } - return out - } - for m, s := range p.Modules { - if s == nil { - continue - } - out[m] = appendNew(out[m], s.First...) - for n := range s.Rest { - out[m] = appendNew(out[m], n) - } - carried(s.Gate) - if len(out[m]) == 0 { - delete(out, m) - } - } - return out -} - -// touchesOf turns what a walk moves, per module, into what it touches, per machine: the modules it moves there, -// and, where it moves the tool runner, every module there whose tools it serves, and where it moves the -// node-engine, every module there. Sorted by machine and module. -func touchesOf(moved map[string][]string, on map[string][]string, shelf map[string]catalogue.Manifest) []walkTouch { - byMachine := map[string][]string{} - for m, machines := range moved { - for _, n := range machines { - byMachine[n] = appendNew(byMachine[n], m) - } - } - out := make([]walkTouch, 0, len(byMachine)) - for n, modules := range byMachine { - t := walkTouch{Machine: n, Modules: modules} - engine, runner := slices.Contains(modules, hostModule), slices.Contains(modules, broker.RuntimeModule) - if engine || runner { - for _, m := range on[n] { - if slices.Contains(t.Modules, m) || (!engine && !servesTools(shelf[m])) { - continue - } - t.Modules = append(t.Modules, m) - t.Carried = append(t.Carried, m) - } - } - sort.Strings(t.Modules) - sort.Strings(t.Carried) - out = append(out, t) - } - sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine }) - return out -} - -// servesTools says the tool runner serves something of this module: a bundle it loads entrypoints from, or the -// module's own tools. -func servesTools(m catalogue.Manifest) bool { - if len(m.Tools) > 0 { - return true - } - for _, b := range m.Bundles { - if len(b.Loads) > 0 { - return true - } - } - return false -} - -// walkRead is the facts a walk's touches are worked out from, read once per event. -type walkRead struct { - running func(string) []string - rollsOut func(string) bool - onMachine func(moved map[string][]string) map[string][]string - shelf func() map[string]catalogue.Manifest -} - -// readWalkFacts reads what the walk's touches need. Without facts (a test, a command with no store), nothing runs -// anywhere and every module rolls out: the touches are then what the walk itself recorded. -func readWalkFacts(ctx context.Context, facts walkFacts, p inventory.Plan) walkRead { - cantRead := func(what string, err error) { - fmt.Fprintf(os.Stderr, "%s: %s cannot be read for what the walk touches: %v\n", p.ID, what, err) - } - r := walkRead{ - running: func(string) []string { return nil }, - rollsOut: func(string) bool { return true }, - onMachine: func(map[string][]string) map[string][]string { return nil }, - shelf: func() map[string]catalogue.Manifest { return nil }, - } - if facts == nil { - return r - } - running := map[string][]string{} - r.running = func(m string) []string { - if on, read := running[m]; read { - return on - } - on, err := facts.Running(ctx, m) - if err != nil { - cantRead("where "+m+" runs", err) - } - running[m] = on - return on - } - rolls := map[string]bool{} - r.rollsOut = func(m string) bool { - if yes, read := rolls[m]; read { - return yes - } - u, err := facts.UpgradeOf(ctx, m) - if err != nil { - cantRead(m+"'s policy", err) - } - // A policy that cannot be read is taken to roll out: the walk is then held more, never less. - rolls[m] = err != nil || u.RollOut - return rolls[m] - } - r.onMachine = func(moved map[string][]string) map[string][]string { - out := map[string][]string{} - for _, core := range []string{hostModule, broker.RuntimeModule} { - for _, n := range moved[core] { - if _, read := out[n]; read { - continue - } - var there []string - if sent, known, err := facts.SentBuilds(ctx, n); err != nil { - cantRead("what "+n+" was sent", err) - } else if known { - for m := range sent { - there = append(there, m) - } - } - if assigned, err := facts.Assigned(ctx, n); err != nil { - cantRead("what "+n+" is assigned", err) - } else { - there = appendNew(there, assigned...) - } - sort.Strings(there) - out[n] = there - } - } - return out - } - r.shelf = func() map[string]catalogue.Manifest { - shelf, err := facts.Catalogue(ctx) - if err != nil { - cantRead("the catalogue", err) - } - return shelf - } - return r -} diff --git a/cmd/mesh-controller/walk_said_integration_test.go b/cmd/mesh-controller/walk_said_integration_test.go deleted file mode 100644 index 52f09231..00000000 --- a/cmd/mesh-controller/walk_said_integration_test.go +++ /dev/null @@ -1,285 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "os" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - cut := time.Now().UTC() - waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", - Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, - Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - touches, _ := json.Marshal(started["touches"]) - if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` || - started["budget_class"] != inventory.ClassLeaf || - started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) { - t.Fatalf("the start: %v", started) - } - if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - touches, _ = json.Marshal(ended["touches"]) - if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` { - t.Fatalf("the end, which sent nothing: %v", ended) - } -} - -func planMovedHeard(t *testing.T) func(phase string) map[string]any { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 64) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - return func(phase string) map[string]any { - t.Helper() - deadline := time.After(5 * time.Second) - for { - select { - case m := <-heard: - var said map[string]any - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said["phase"] == phase { - return said - } - case <-deadline: - t.Fatalf("no walk was said %s", phase) - } - } - } -} - -func TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - for _, verb := range []string{"stop", "close"} { - cut := time.Now().UTC() - walk := inventory.Plan{ID: "plan-308-" + verb, Repository: "novox/mesh-catalog", Branch: "main", - Commit: "c8c8c8c8", Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Times: &inventory.PlanTimes{Cut: &cut, Started: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{verb, walk.ID, "--why", "a test"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkStopped { - t.Fatalf("plans %s: the end is %v", verb, ended) - } - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err != nil { - t.Fatal(err) - } - if p.Outcome() != inventory.WalkStopped || p.Delivery == nil || p.Delivery.StoppedWhy != "a test" { - t.Fatalf("plans %s kept the walk %s, %+v", verb, p.Outcome(), p.Delivery) - } - } -} - -func TestAWalkRetriedByAPersonIsSaidStartedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - next := planMovedHeard(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-308-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: "killed by hand"}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{"retry", failed.ID}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if started["walk"] != failed.ID || started["budget_class"] != inventory.ClassLeaf { - t.Fatalf("the retried start: %v", started) - } -} - -func TestABacklogReleasedByAPersonIsSaidStarted(t *testing.T) { - b := aBacklog(t) - next := planMovedHeard(t) - if err := backlogCommand(t.Context(), "release-backlog", []string{"--why", "a test"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if p := b.release(t); started["walk"] != p.ID { - t.Fatalf("the release's start: %v", started) - } -} - -func asTyped(t *testing.T, args ...string) error { - t.Helper() - was := os.Args - os.Args = append([]string{"mesh-controller"}, args...) - t.Cleanup(func() { os.Args = was }) - err := run() - // run gives back the lease as the process would end; the test's process goes on, and takes it again. - theLease.mu.Lock() - theLease.held, theLease.kv, theLease.noBus = nil, nil, false - theLease.mu.Unlock() - return err -} - -func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - twoTiers(t, open) - for _, again := range [][]string{{"plans", "retry"}} { - before := time.Now().UTC().Add(-time.Hour) - walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before) - if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil { - t.Fatal(err) - } - // The build asked before the stop fails after it. - p, _ := open.inventory.PlanByID(ctx, walk.ID) - p.Modules["a"].State, p.Modules["a"].Why = "failed", "it broke" - if err := open.inventory.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - args := again - if again[0] == "plans" { - args = append(append([]string(nil), again...), walk.ID) - } - if err := asTyped(t, args...); err != nil { - t.Fatal(err) - } - p, _ = open.inventory.PlanByID(ctx, walk.ID) - if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") { - t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery) - } - // The retry takes a from its key (it asks no build), at the commit the walk carries. - if a := p.Modules["a"]; a == nil || !a.Key || a.KeyAt != "c0ffee" { - t.Fatalf("%s: the retried walk does not take a from its key: %+v", again[0], a) - } - // The key fails again: the builder's record says so, and the look settles the walk from it. - wholeKeys := openPlanKeys - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysFailing{all, "a", "it broke again"}, done, err - } - t.Cleanup(func() { openPlanKeys = wholeKeys }) - advancePlans(ctx, open) - if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed { - t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome()) - } - } -} - -// novox/hq ADR 0505 §2: a walk's start is said again, over the real bus and against the real store, with a machine it -// sends nothing to whose declaration differs from what it was last sent (here: assigned a module and never pushed), -// marked declared. -func TestAWalksStartIsSaidAgainWithAMachineWhoseDeclarationChanged(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - core, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-505", State: inventory.PlanBuilding}, - walkSaid: &walkSaid{Walk: "plan-505", Phase: inventory.WalkStarted, BudgetClass: inventory.ClassLeaf, - BoundSeconds: 360, Touches: []walkTouch{}}}) - if err != nil { - t.Fatal(err) - } - sayDeclared(ctx, link.OverNATS{Conn: js.Conn(), JS: core}, open, body) - again := next(inventory.WalkStarted) - raw, _ := json.Marshal(again["touches"]) - var touches []walkTouch - _ = json.Unmarshal(raw, &touches) - anchor := false - for _, tc := range touches { - anchor = anchor || (tc.Machine == "anchor" && tc.Declared && len(tc.Modules) == 0) - if !tc.Declared { - t.Fatalf("a machine the walk sends nothing to was said sent: %s", raw) - } - } - if again["walk"] != "plan-505" || again[walkDeclaredRead] != true || !anchor { - t.Fatalf("said again: %v", again) - } -} - -// keysFailing is the stand-in builder's keys but for one module, whose key is failed in these words. -type keysFailing struct { - planKeyReads - module string - words string -} - -func (k keysFailing) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) { - if module == k.module { - return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: k.words}}, nil - } - return k.planKeyReads.Build(ctx, module, fp) -} diff --git a/cmd/mesh-controller/walk_said_test.go b/cmd/mesh-controller/walk_said_test.go deleted file mode 100644 index f279146a..00000000 --- a/cmd/mesh-controller/walk_said_test.go +++ /dev/null @@ -1,269 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "reflect" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// fakeWalkFacts is the inventory as far as a walk's touches read it. -type fakeWalkFacts struct { - running map[string][]string - records map[string]bool // modules whose policy records rather than rolls out - sent map[string]map[string]string - assigned map[string][]string - shelf map[string]catalogue.Manifest -} - -func (f fakeWalkFacts) Running(_ context.Context, m string) ([]string, error) { - return f.running[m], nil -} -func (f fakeWalkFacts) UpgradeOf(_ context.Context, m string) (inventory.Upgrade, error) { - return inventory.Upgrade{RollOut: !f.records[m]}, nil -} -func (f fakeWalkFacts) Assigned(_ context.Context, n string) ([]string, error) { - return f.assigned[n], nil -} -func (f fakeWalkFacts) SentBuilds(_ context.Context, n string) (map[string]string, bool, error) { - s, ok := f.sent[n] - return s, ok, nil -} -func (f fakeWalkFacts) Catalogue(context.Context) (map[string]catalogue.Manifest, error) { - return f.shelf, nil -} - -// theMesh is four machines: the tool runner on each, messenger and mesh-issues on novox, gitea on ace. -func theMesh() fakeWalkFacts { - return fakeWalkFacts{ - running: map[string][]string{ - broker.RuntimeModule: {"ace", "g14", "novox", "shanks"}, - "messenger": {"novox"}, - "gitea": {"ace"}, - "plex": {"ace"}, - }, - sent: map[string]map[string]string{ - "novox": {broker.RuntimeModule: "c0", "messenger": "c1", "mesh-issues": "c2", "postgres": "c3"}, - "ace": {broker.RuntimeModule: "c0", "gitea": "c4"}, - }, - assigned: map[string][]string{"g14": {broker.RuntimeModule, "desktop"}, "shanks": {broker.RuntimeModule}}, - shelf: map[string]catalogue.Manifest{ - "messenger": {Module: "messenger", Tools: []string{"notify"}}, - "mesh-issues": {Module: "mesh-issues", Bundles: []catalogue.Bundle{{Name: "tools", Loads: []string{"main"}}}}, - "postgres": {Module: "postgres"}, - "gitea": {Module: "gitea", Tools: []string{"gitea_api"}}, - "desktop": {Module: "desktop", Tools: []string{"notify_send"}}, - }, - } -} - -func walkOfTiers(class string, tiers ...[]string) inventory.Plan { - cut := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC) - started := cut.Add(time.Second) - p := inventory.Plan{ID: "plan-308", State: inventory.PlanBuilding, Tiers: tiers, - Modules: map[string]*inventory.PlanModule{}, Times: &inventory.PlanTimes{Cut: &cut, Class: class, Started: &started}} - for _, tier := range tiers { - for _, m := range tier { - p.Modules[m] = &inventory.PlanModule{} - } - } - p.WalkSaid = inventory.WalkStarted - return p -} - -// novox/hq ADR 0308 How it is checked §1: the start names the modules the walk moves and the machines it sends -// them to, and every module a moved tool runner carries there. -func TestAWalksStartNamesWhatItTouchesWithWhatTheToolRunnerCarries(t *testing.T) { - p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"}) - w := sayWalk(t.Context(), theMesh(), p) - if w == nil || w.Walk != "plan-308" || w.Phase != inventory.WalkStarted || w.Outcome != "" { - t.Fatalf("the start is said: %+v", w) - } - want := []walkTouch{ - {Machine: "ace", Modules: []string{"gitea", broker.RuntimeModule}, Carried: []string{"gitea"}}, - {Machine: "g14", Modules: []string{"desktop", broker.RuntimeModule}, Carried: []string{"desktop"}}, - // postgres serves no tools: the tool runner does not carry it. - {Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule}, Carried: []string{"mesh-issues"}}, - {Machine: "shanks", Modules: []string{broker.RuntimeModule}}, - } - if !reflect.DeepEqual(w.Touches, want) { - t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want) - } - if w.BudgetClass != inventory.ClassCore || w.BoundSeconds != 1200 || w.StartedAt == nil { - t.Fatalf("a core walk is bound at 20 minutes: %+v", w) - } -} - -// A moved node-engine carries every module on its machine; a module whose policy records is not sent, and not -// touched. -func TestAMovedNodeEngineCarriesEverythingOnItsMachine(t *testing.T) { - f := theMesh() - f.running[hostModule] = []string{"novox"} - f.records = map[string]bool{"plex": true} - p := walkOfTiers(inventory.ClassCore, []string{hostModule, "plex"}) - w := sayWalk(t.Context(), f, p) - want := []walkTouch{{Machine: "novox", - Modules: []string{"mesh-host", "mesh-issues", "messenger", "node-tools", "postgres"}, - Carried: []string{"mesh-issues", "messenger", "node-tools", "postgres"}}} - if !reflect.DeepEqual(w.Touches, want) { - t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want) - } -} - -// The end names what the walk reached — its first machines, its rest, what a gated send carried — and its -// outcome; the modules a reached tool runner carries are touched there too. -func TestAWalksEndNamesWhatItReachedAndItsOutcome(t *testing.T) { - p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"}) - ended := p.Times.Started.Add(4 * time.Minute) - p.Times.Ended = &ended - p.State, p.WalkSaid = inventory.PlanDone, inventory.WalkEnded - p.Modules[broker.RuntimeModule] = &inventory.PlanModule{First: []string{"shanks"}, - Rest: map[string]inventory.SentDeclaration{"novox": {}}, - Gate: &inventory.PlanGate{Carried: []inventory.CarriedMove{{Module: "gitea", Node: "shanks"}}}} - // messenger was built and found unchanged: no move, not reached. - p.Modules["messenger"] = &inventory.PlanModule{Why: "no move"} - w := sayWalk(t.Context(), theMesh(), p) - if w.Phase != inventory.WalkEnded || w.Outcome != inventory.WalkDelivered || w.EndedAt == nil { - t.Fatalf("the end is said with its outcome: %+v", w) - } - want := []walkTouch{ - {Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule}, - Carried: []string{"mesh-issues", "messenger"}}, - {Machine: "shanks", Modules: []string{"gitea", broker.RuntimeModule}}, - } - if !reflect.DeepEqual(w.Touches, want) { - t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want) - } - for outcome, end := range map[string]func(*inventory.Plan){ - inventory.WalkFailed: func(p *inventory.Plan) { p.State = inventory.PlanFailed }, - inventory.WalkStopped: func(p *inventory.Plan) { - p.State, p.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} - }, - } { - q := p - end(&q) - if got := sayWalk(t.Context(), theMesh(), q).Outcome; got != outcome { - t.Errorf("outcome %q, want %q", got, outcome) - } - } -} - -// novox/hq ADR 0308 How it is checked §1 and decision 5: the bound is twice the largest budget in force among -// the walk's deliveries — 20 minutes for a core walk, twice the leaf budget for a leaf walk — and 15 minutes, not -// doubled, for a walk none of whose deliveries has a budget. -func TestAWalksBoundIsTwiceItsBudgetOrFifteenMinutesWithNone(t *testing.T) { - f := theMesh() - f.records = map[string]bool{"plex": true} - release := walkOfTiers("", []string{"gitea"}) - release.Times, release.Release = &inventory.PlanTimes{}, &inventory.PlanRelease{Order: []string{"ace"}} - for name, c := range map[string]struct { - p inventory.Plan - class string - bound int - }{ - "core": {walkOfTiers(inventory.ClassCore, []string{"messenger"}), inventory.ClassCore, 1200}, - "leaf": {walkOfTiers(inventory.ClassLeaf, []string{"messenger"}), inventory.ClassLeaf, int(2 * deliveryBudgets[inventory.ClassLeaf] / time.Second)}, - "held for a person": {walkOfTiers(inventory.ClassLeaf, []string{"plex"}), budgetNone, 900}, - "a release": {release, budgetNone, 900}, - "kept without a class": {walkOfTiers("", []string{"messenger"}), budgetNone, 900}, - } { - w := sayWalk(t.Context(), f, c.p) - if w.BudgetClass != c.class || w.BoundSeconds != c.bound { - t.Errorf("%s: %s bound %ds, want %s %ds", name, w.BudgetClass, w.BoundSeconds, c.class, c.bound) - } - } - // The leaf budget in force is three minutes (ADR 0304 decision 4): a leaf walk is bound at six. - if deliveryBudgets[inventory.ClassLeaf] == 3*time.Minute { - if w := sayWalk(t.Context(), f, walkOfTiers(inventory.ClassLeaf, []string{"messenger"})); w.BoundSeconds != 360 { - t.Errorf("a leaf walk at a three-minute budget is bound at 360 s: %d", w.BoundSeconds) - } - } -} - -// plan-moved's body: the walk whole as before, and on its start the walk's fields beside it; on any other save -// none of them. -func TestPlanMovedCarriesTheWalkWholeAndOnItsStartWhatItTouches(t *testing.T) { - p := walkOfTiers(inventory.ClassLeaf, []string{"messenger"}) - body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)}) - if err != nil { - t.Fatal(err) - } - var said map[string]any - if err := json.Unmarshal(body, &said); err != nil { - t.Fatal(err) - } - for _, k := range []string{"id", "state", "tiers", "modules", "times", "walk", "phase", "touches", "bound_seconds", "budget_class", "started_at"} { - if _, ok := said[k]; !ok { - t.Errorf("plan-moved lacks %q: %s", k, body) - } - } - if _, ok := said["outcome"]; ok { - t.Errorf("a start says no outcome: %s", body) - } - if said["walk"] != "plan-308" || said["phase"] != "started" || said["budget_class"] != "leaf" { - t.Errorf("the walk's fields: %s", body) - } - p.WalkSaid = "" - body, _ = json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)}) - said = map[string]any{} - _ = json.Unmarshal(body, &said) - for _, k := range []string{"walk", "phase", "touches", "bound_seconds"} { - if _, ok := said[k]; ok { - t.Errorf("a save neither starting nor ending the walk says %q: %s", k, body) - } - } - // A walk that touches nothing says an empty list, never null. - q := walkOfTiers(inventory.ClassLeaf, []string{"nobody-runs-this"}) - body, _ = json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), theMesh(), q)}) - said = map[string]any{} - _ = json.Unmarshal(body, &said) - if touches, ok := said["touches"].([]any); !ok || len(touches) != 0 { - t.Errorf("touches nothing: %s", body) - } -} - -// Live, with the store and a bus: a walk let go by the delivery's verb is said started with what it touches and -// its bound, and stopped by the verb is said ended, stopped (novox/hq ADR 0308 How it is checked §1). - -// planMovedHeard raises a bus of the test's own, points the verbs' hand-act connection at it, and returns what -// waits for the next plan-moved said with the given phase, failing the test when none comes within 5 s. - -// Live, with the store and a bus: a walk a person stops with `plans stop` is said ended, outcome stopped, by the -// verb itself, and is kept stopped by that person; `plans close` ends one the same way (novox/hq ADR 0308 -// decision 1, review 15 on mesh-controller#247). - -// Live, with the store and a bus: a failed walk retried with `plans retry` starts again, and the verb says its -// start (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). - -// Live, with the store and a bus: the release plan `upgrade release-backlog` opens is said started by the verb -// (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). - -// asTyped runs a verb as the controller's command line runs it, through run, as a person's terminal would. - -// aStartedWalkAsking keeps a live walk, said started, whose first tier asked a as build. -func aStartedWalkAsking(t *testing.T, open *stores, id, build string, asked time.Time) inventory.Plan { - t.Helper() - walk := inventory.Plan{ID: id, Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: asked, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: build}}, - Times: &inventory.PlanTimes{Cut: &asked, Started: &asked, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(t.Context(), &walk); err != nil { - t.Fatal(err) - } - return walk -} - -// Live, with the store and a bus: `cancel ` and `clear` fail the walk that asked the build, and the verb -// says its end, failed (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). - -// Live, with the store and a bus: `rebuild ` joining a failed walk sets it building again, and the verb -// says its start (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). - -// Live, with the store: a walk a person stopped and then retried, by `plans retry` or by `rebuild`, walks again -// without the stop, so when it fails again it ends failed, not stopped, and the search for the merge that -// brought the failure is not kept from it (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). diff --git a/cmd/mesh-controller/walk_wait_rebuild_754_test.go b/cmd/mesh-controller/walk_wait_rebuild_754_test.go deleted file mode 100644 index 97b134cb..00000000 --- a/cmd/mesh-controller/walk_wait_rebuild_754_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 754: a tier partly asked asks the rest; a module held built that the plan never asked — the live -// shape, a hand rebuild taken in during the wait, built after the plan was made — is asked again; one asked is kept. -func TestATierPartlyAskedAsksTheRest(t *testing.T) { - created := time.Date(2026, 10, 14, 23, 30, 0, 0, time.UTC) - earlier, later := created.Add(-time.Hour), created.Add(7*time.Minute) - p := inventory.Plan{Created: created, Tiers: [][]string{{"build-agent", "gitea", "route-proxy", "stale", "kept", "own"}}, - Modules: map[string]*inventory.PlanModule{ - "gitea": {}, - "route-proxy": {State: "built", BuiltAt: &later, Commit: "c51357f8"}, - "stale": {State: "built", BuiltAt: &earlier}, - "kept": {State: "asked", AskedAt: &created, Build: "build-kept"}, - "own": {State: "built", AskedAt: &created, BuiltAt: &later, Build: "build-own"}, - }} - if got, want := unaskedOf(p), []string{"build-agent", "gitea", "route-proxy", "stale"}; !reflect.DeepEqual(got, want) { - t.Fatalf("unasked %v, want %v", got, want) - } - if !askedInTier(p) { - t.Fatal("a tier with a module asked under an id reads as never asked") - } - p.Modules["kept"], p.Modules["own"] = nil, nil - if askedInTier(p) { - t.Fatal("a tier the plan never asked reads as asked") - } -} - -// novox/hq issue 754, as seen on 2026-10-14: a rebuild lands while the walk waits for its word; the walk is let go; -// it asks every module of its tier, and completes it once those builds come in. diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 208c98da..cdce1875 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "slices" - "sort" "sync" "time" @@ -49,25 +48,9 @@ type signalFacts struct { // since then at the most. toolsHeardFrom time.Time - plans []planFacts - plansErr error - // waits are the walks waiting for their delivery's word (S16, novox/hq ADR 0239). - waits []waitFacts - // batches are the batches not yet cut (S18, S19, novox/hq ADR 0276). - batches []batchFacts - // failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449). - failed []failedPlanFacts - failedErr error - // held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732). - held []heldFacts - heldErr error // images are the modules left out of a send because the artifact store lacks an image they name (S25, issue 1742). images []imageFacts imagesErr error - // refusedSends are the sends a machine refused within the hour for the generation they were composed - // from, each naming its sender (S20, novox/hq issue 234). - refusedSends []inventory.Send - refusedSendsErr error // doing is every machine whose last report was not a clean apply, with since when it has said the same // failure (S21, novox/hq issue 233). doing []inventory.Doing @@ -120,9 +103,6 @@ type signalFacts struct { // facts is the snapshot this controller keeps for merge checks (S14). facts factsFacts - - bus busFacts - busErr error } // factsFacts is when the newest facts snapshot was taken, when this controller began keeping it, and @@ -166,25 +146,6 @@ type machineFacts struct { // asleep says the machine said it would be away and has not said it is back (ADR 0211). func (m machineFacts) asleep() bool { return m.power.Away() } -type planFacts struct { - id, repository, commit string - tier, tiers int - entered time.Time - bound time.Duration - waiting string - bus bool -} - -// waitFacts is one walk waiting for its delivery's word: since its merge opened it. -type waitFacts struct { - id, repository, commit, awaits string - since time.Time - // modules are the modules the walk moves, as a person names the delivery. - modules []string - // merges are the merges it answers (novox/hq ADR 0276), as the deliveries to read are found. - merges []inventory.PlanMerge -} - type loopFacts struct { took time.Time pending uint64 @@ -347,13 +308,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } } f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now) - f.bus, f.busErr = gatherBus(ctx, inv) - f.plans, f.waits, f.plansErr = gatherPlans(ctx, inv, now, f.bus.heldByTheBus()) - if f.plansErr == nil { - f.batches, f.plansErr = gatherBatches(ctx, inv, now) - } - f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now) - f.held, f.heldErr = gatherHeldMoves(ctx, inv) f.images, f.imagesErr = gatherMissingImagesHere(ctx, inv) f.loop, f.loopErr = w.gatherLoop() if open, err := w.keeper.Open(ctx); err != nil { @@ -390,7 +344,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { f.deadLettersAging, f.advisoriesErr = link.AgingDeadLetters(w.js.Context(), f.deadLetters) } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) - f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx) if f.doingErr == nil && w.js != nil { f.provFailed, f.doingErr = provisioningFailed(w.js.Context()) @@ -427,7 +380,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory if err != nil { return nil, fmt.Errorf("the machines cannot be read: %w", err) } - reports, err := inv.LastReports(ctx) + reports, err := lastReports(ctx, inv) if err != nil { return nil, fmt.Errorf("what each machine last reported cannot be read: %w", err) } @@ -443,14 +396,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory if err != nil { return nil, fmt.Errorf("where the node tools run cannot be read: %w", err) } - applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour)) - if err != nil { - return nil, fmt.Errorf("the measured applies cannot be read: %w", err) - } lastApply := map[string]time.Duration{} - for _, d := range applies { // oldest first: the last one read is the newest - lastApply[d.Node] = d.Took - } var out []machineFacts anyPast := false for _, n := range nodes { @@ -504,118 +450,6 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory return out, nil } -// gatherBus is a new bus build waiting for its step and the walks refused for it (S17, novox/hq issue 336). -func gatherBus(ctx context.Context, inv *inventory.Inventory) (busFacts, error) { - b, err := pendingBus(ctx, inv) - if err != nil { - return busFacts{}, fmt.Errorf("what a bus upgrade would do cannot be read: %w", err) - } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return busFacts{}, fmt.Errorf("the open plans cannot be read: %w", err) - } - f := busWaitsOf(plans, b, busRefusedFirst.of) - busRefusedFirst.keepOnly(f.heldByTheBus()) - return f, nil -} - -// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on; byTheBus are -// the walks held only by the bus's planned step, which S3 leaves to S17. -func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time, byTheBus map[string]bool) ([]planFacts, []waitFacts, error) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, nil, fmt.Errorf("the open plans cannot be read: %w", err) - } - if len(plans) == 0 { - return nil, nil, nil - } - bounds, err := measuredTierBounds(ctx, inv, now) - if err != nil { - return nil, nil, err - } - var out []planFacts - var waits []waitFacts - for _, p := range plans { - // A walk waiting for its delivery's word is no tier late (novox/hq ADR 0239); how long it has waited is - // S16's, whatever mesh-delivery says or does not say. - if p.Waiting() { - waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit, - awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p), merges: p.Delivery.Merges}) - continue - } - // A walk let go and waiting for the walk before it to end (ADR 0276) is no tier late either: the walk - // before it is the one S3 watches. - if deferred(p) { - continue - } - // A walk waiting on, or held by, its backups is no tier late either: the backup condition is the one voice - // (novox/hq ADR 0404 decision 4, issue 1066). - if waitingOnBackups(p) { - continue - } - bound := bounds.of(p.Repository) - out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier, - tiers: len(p.Tiers), entered: inTierSince(p), bound: bound, - waiting: planLineWith(p, now, bound), bus: byTheBus[p.ID]}) - } - return out, waits, nil -} - -// tierBounds is how long a plan's tier may take before it is stalled (S3) and `plans` and `status` call -// it LATE, by repository: three times the ninetieth percentile of the tiers measured for it, and never -// less than tierAtLeast. One bound for the watchdog and the lines a person reads, so the two cannot -// disagree about one plan (novox/hq issue 296). A repository nothing was measured for, or a nil -// tierBounds, is given tierAtLeast. -type tierBounds map[string]time.Duration - -func (b tierBounds) of(repository string) time.Duration { - if d, ok := b[repository]; ok { - return d - } - return tierAtLeast -} - -// measuredTierBounds is the tier bounds from the last fourteen days of measured plan tiers. -func measuredTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) (tierBounds, error) { - tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour)) - if err != nil { - return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err) - } - return boundsOf(tiers), nil -} - -// readTierBounds is measuredTierBounds for a line a person reads: what cannot be read is said, and -// every plan is then given the least bound, as the watchdog gives a repository nothing was measured for. -func readTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) tierBounds { - bounds, err := measuredTierBounds(ctx, inv, now) - if err != nil { - fmt.Fprintf(os.Stderr, "%v; a plan is called late past %s\n", err, tierAtLeast) - } - return bounds -} - -func boundsOf(tiers []inventory.Duration) tierBounds { - measured := map[string][]time.Duration{} - for _, d := range tiers { - measured[d.Subject] = append(measured[d.Subject], d.Took) - } - out := tierBounds{} - for repository, took := range measured { - out[repository] = max(tierAtLeast, 3*p90(took)) - } - return out -} - -// p90 is the ninetieth percentile of measurements; zero for none. -func p90(took []time.Duration) time.Duration { - if len(took) == 0 { - return 0 - } - sorted := append([]time.Duration(nil), took...) - sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) - return sorted[int(0.9*float64(len(sorted)-1))] -} - // loopConsumers are the controller's own consumers (issue 1063). func loopConsumers() []broker.Consumer { return broker.ControllerConsumers() } diff --git a/cmd/mesh-controller/worker_order_test.go b/cmd/mesh-controller/worker_order_test.go deleted file mode 100644 index 86fb9073..00000000 --- a/cmd/mesh-controller/worker_order_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The holder of the build seat follows the controller that defines its worker (novox/hq issue 206). -// On 2026-10-03 a plan put the build machine in tier 0 and the controller in tier 1; the new build -// machine could not bind the worker the old controller had defined, and nothing could build the -// controller that would have redefined it. The built-by edge from the controller to its build -// machine yields to that order: the controller is built by whichever build machine is running. -func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.T) { - edges := []inventory.Edge{ - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgePackages}, - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - {From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages}, - {From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - } - // A packages edge recorded before novox/hq ADR 0267 widens nothing: the controller moved alone moves - // alone, and a change to a package all three build from moves all three, each by its own build source. - if alone := reachableFrom([]string{"mesh-controller"}, edges); len(alone) != 1 { - t.Fatalf("the controller alone, whatever packages its repository: %v", alone) - } - set := reachableFrom([]string{"mesh-controller", "build-agent", "route-proxy"}, edges) - tiers := tiersOf(set, edges) - pos := map[string]int{} - for i, tier := range tiers { - for _, m := range tier { - pos[m] = i - } - } - if pos["mesh-controller"] != 0 { - t.Fatalf("the controller first, built by the build machine that is running: %v", tiers) - } - if pos["build-agent"] <= pos["mesh-controller"] { - t.Fatalf("the build machine after the controller that defines its worker: %v", tiers) - } - // Built-by orders nothing (novox/hq ADR 0360 §3): whichever builder runs builds the proxy, beside the controller. - if pos["route-proxy"] != 0 { - t.Fatalf("what the builder builds waits for a new one: %v", tiers) - } - if hasCycle(tiers, edges) { - t.Fatalf("no cycle here: %v", tiers) - } -} diff --git a/go.mod b/go.mod index 4e0a9bd9..a6bd6973 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 6b35a21b..1f380c29 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,7 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4= git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/broker/agreement_catalogue_test.go b/internal/broker/agreement_catalogue_test.go index 96eba076..8635bf39 100644 --- a/internal/broker/agreement_catalogue_test.go +++ b/internal/broker/agreement_catalogue_test.go @@ -179,6 +179,15 @@ func TestARetiredSeatsConsumeIsSetAsideAndAnUnknownOneIsNot(t *testing.T) { // nothing is silence. func TestTheCataloguesDerivedSubjectsMeet(t *testing.T) { emitters, consumers, seats := theCataloguesEvents(t) + installed := map[string]bool{} // the modules of this catalogue, whether they emit or not + root := beside.Catalogue(t) + if dirs, err := os.ReadDir(root); err == nil { + for _, d := range dirs { + if _, err := os.Stat(filepath.Join(root, d.Name(), "module.json")); err == nil { + installed[d.Name()] = true + } + } + } // Every subject something publishes: a module's own events, and the events of every role. published := map[string]bool{} @@ -229,6 +238,14 @@ func TestTheCataloguesDerivedSubjectsMeet(t *testing.T) { if strings.Contains(subject, "*") || strings.Contains(subject, ">") { continue } + // And a module that lives in another repository (mesh-delivery's view and its kin) is not installed here, so its + // events cannot be seen here: excused, as the comment above says, unless the name is a role's, which is the very + // mistake this test exists for. + if module, _, ok := strings.Cut(strings.TrimPrefix(subject, "mesh.mod."), ".event."); ok && strings.HasPrefix(subject, "mesh.mod.") { + if _, isASeat := byName[module]; !isASeat && !installed[module] { + continue + } + } lonely = append(lonely, c.Module+" subscribes "+subject+", which nothing publishes") } } diff --git a/internal/broker/announced.go b/internal/broker/announced.go index e05f7835..1c8304a9 100644 --- a/internal/broker/announced.go +++ b/internal/broker/announced.go @@ -17,8 +17,8 @@ import ( const StatesAnnouncedBucket = announce.KeptBucket // controllerReadable are the controller's states others may read: the record mirrors (the packages each node declares -// among them, which the pacman module reads, ADR 0507 §5), the chain's buckets and the grants issued, which the assigner -// reads (ADR 0528 clause 2). +// among them, which the pacman module reads, ADR 0507 §5), the chain's buckets, and the grants issued and the +// compositions' digests, which the assigner reads (ADR 0528 clause 2, ADR 0539 clause 6). var controllerReadable = []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, MirrorDeclaredPackages, ControllerGrants, ControllerComposed} diff --git a/internal/broker/announced_test.go b/internal/broker/announced_test.go index ee4a600a..ccb4fdb0 100644 --- a/internal/broker/announced_test.go +++ b/internal/broker/announced_test.go @@ -44,11 +44,17 @@ func TestTheControllerAnnouncesWhatItKeeps(t *testing.T) { if !slices.Equal(readable, want) { t.Fatalf("readable %v, want %v", readable, want) } - for _, name := range []string{"lease", "calls", "states-announced", "merge-window"} { + for _, name := range []string{"lease", "calls", "states-announced"} { if !s.Keeps(name) || s.Readable(name) { t.Errorf("%s: kept %v, readable %v", name, s.Keeps(name), s.Readable(name)) } } + // The buckets of plan state release 6 deleted are no longer kept (ADR 0539 clause 13). + for _, name := range []string{"merge-window", "rollout-silent", "plans-building"} { + if s.Keeps(name) { + t.Errorf("%s is still kept", name) + } + } perms, err := PermissionsFor(Principal{Kind: KindController}) if err != nil { t.Fatal(err) diff --git a/internal/broker/composed.go b/internal/broker/composed.go index 8bfb0634..da5c495a 100644 --- a/internal/broker/composed.go +++ b/internal/broker/composed.go @@ -8,57 +8,105 @@ import ( "github.com/nats-io/nats.go/jetstream" ) -// Release 6's grants, carried ahead of it (novox/hq ADR 0539 clauses 2, 6, 9 and 16; issue 1945, the review's A1). The -// switch to release 6 is the last delivery plan of today's controller, and the bus's user list after it is the one -// today's controller composes: so the subjects the new controller, every new node-engine and the assigner use after the -// switch are granted by this controller already, before the group merges. Unused until then: nothing writes or reads -// these buckets before release 6. -// -// Names as mesh-sdk's release 6 gives them (delivery.ComposedBucketFor, ControllerComposedBucket); written out here -// because this controller's mesh-sdk predates them, and held equal by release 6's own tests. +// The compositions' buckets (novox/hq ADR 0539 clauses 2 and 4): one per node, `composed-`, holding that node's +// compositions, which only that node's node-engine reads, since a composition can carry a sealed value (ADR 0488 §2's +// reason); and `mesh-controller_composed`, holding each composition's digest and the judgement of its move, which the +// assigner reads. The controller writes both, by compare-and-set while it holds the lease; each keeps five revisions +// of a key. The names and layouts are mesh-sdk's (delivery.ComposedBucketFor, ControllerComposedBucket, ComposedKey). -// ComposedBucketPrefix starts a node's bucket of compositions, `composed-`. -const ComposedBucketPrefix = "composed-" - -// ControllerComposed is the local name of the compositions' digests, `mesh-controller_composed`, which the assigner reads. +// ControllerComposed is the local name of the compositions' digests, as a reader names it in `reads` after +// `mesh-controller.`: delivery.ControllerComposedBucket. const ControllerComposed = "composed" -// ControllerComposedBucket is the bucket of the compositions' digests. -var ControllerComposedBucket = BucketName(ControllerSeat, ControllerComposed) +// ComposedValueBytes is the most a composition may hold: a file over 64 KiB travels by digest, so a module's +// composition is far smaller; just under the server's largest message, which also carries the subject and headers. +const ComposedValueBytes = 1<<20 - 8<<10 -// composedBucketOf is a node's bucket of compositions; a node name a subject would widen is refused. -func composedBucketOf(node string) (string, error) { - if !safeSubject.MatchString(node) { - return "", fmt.Errorf("%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", node) +// ComposedBucketBytes is a node's composition bucket's size: five revisions of every module's compositions of the node. +const ComposedBucketBytes = 256 << 20 + +// ComposedDigestsBytes is the size of the bucket of digests: a value of a few hundred bytes per node, module and +// fingerprint kept. +const ComposedDigestsBytes = 64 << 20 + +// ComposedConfig is a node's composition bucket as the controller asserts it. +func ComposedConfig(node string) (jetstream.KeyValueConfig, error) { + bucket, err := delivery.ComposedBucketFor(node) + if err != nil { + return jetstream.KeyValueConfig{}, err } - return ComposedBucketPrefix + node, nil + return jetstream.KeyValueConfig{ + Bucket: bucket, + Description: "the compositions of " + node + " (novox/hq ADR 0539): each module's resources for this node at one " + + "fingerprint, and the node's own, signed; written by the controller alone, read by this node's node-engine " + + "alone; deleted when outside the module's five newest fingerprints and named by no want", + History: delivery.ComposedHistory, + MaxValueSize: ComposedValueBytes, + MaxBytes: ComposedBucketBytes, + Storage: jetstream.FileStorage, + }, nil +} + +// ComposedDigestsConfig is the bucket of the compositions' digests as the controller asserts it. +func ComposedDigestsConfig() jetstream.KeyValueConfig { + return jetstream.KeyValueConfig{ + Bucket: delivery.ControllerComposedBucket, + Description: "each composition's digest and the judgement of its move from what the node runs (novox/hq ADR 0539 " + + "clauses 2 and 14): written by the controller alone, read by the assigner", + History: delivery.ControllerComposedHistory, + MaxValueSize: 4 << 10, + MaxBytes: ComposedDigestsBytes, + Storage: jetstream.FileStorage, + } +} + +// EnsureComposed creates the bucket of digests and each named node's composition bucket if absent and brings their +// options to match. An update, never a delete. +func EnsureComposed(ctx context.Context, js jetstream.JetStream, nodes ...string) error { + if _, err := js.CreateOrUpdateKeyValue(ctx, ComposedDigestsConfig()); err != nil { + return fmt.Errorf("asserting bucket %s: %w", delivery.ControllerComposedBucket, err) + } + for _, node := range nodes { + cfg, err := ComposedConfig(node) + if err != nil { + return err + } + if _, err := js.CreateOrUpdateKeyValue(ctx, cfg); err != nil { + return fmt.Errorf("asserting bucket %s: %w", cfg.Bucket, err) + } + } + return nil } // ComposedWrites is what the controller publishes to write a node's compositions: the bucket's own subjects. func ComposedWrites(node string) (string, error) { - bucket, err := composedBucketOf(node) + bucket, err := delivery.ComposedBucketFor(node) if err != nil { return "", err } return "$KV." + bucket + ".>", nil } -// ComposedReads is what a node's node-engine publishes to follow its wants and compositions: every key of its own -// composition bucket, and its own wants (`want..`, `want..node`) in the assigner's bucket; nothing -// of another node's, nor the digests. +// ComposedReads is what a node's node-engine publishes to follow its wants and compositions (ADR 0539 clauses 1, 2 and +// 9): every key of its own composition bucket, and its own wants (`want..` and `want..node`) in the +// assigner's bucket; nothing of another node's, nor the bucket of digests. func ComposedReads(node string) ([]string, error) { - bucket, err := composedBucketOf(node) + bucket, err := delivery.ComposedBucketFor(node) if err != nil { return nil, err } + if !safeSubject.MatchString(node) { + return nil, fmt.Errorf("%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", node) + } return append(readGrants(bucket, ""), readGrants(delivery.AssignerWantsBucket, "want."+node+".>")...), nil } -// VerbsTheNodeAgentAsks are the seat verbs a node's node-engine calls of its own machine while it applies a want: its -// backup holder backs a data module up before it moves, and says when it was (ADR 0539 clause 16). +// VerbsTheNodeAgentAsks are the seat verbs a node's node-engine calls of its own machine while it applies a want (ADR +// 0539 clause 16, ADR 0429 §14): its backup holder backs a data module up before it moves, and says when it was. Asked +// of its own machine only, `mesh.seat..tool..`. var VerbsTheNodeAgentAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}, {Seat: "node-backup", Verb: "backed-up"}} -// NodeAgentAsks are the subjects of VerbsTheNodeAgentAsks for one node: `mesh.seat..tool..`. +// NodeAgentAsks are the subjects of VerbsTheNodeAgentAsks for one node. func NodeAgentAsks(node string) ([]string, error) { if !safeSubject.MatchString(node) { return nil, fmt.Errorf("%q cannot be part of a subject: a permission is a subject pattern, and this would widen it", node) @@ -71,34 +119,13 @@ func NodeAgentAsks(node string) ([]string, error) { } // NodeAgentHears are the events a node's node-engine subscribes while it applies a want: the backup holder's word that a -// backup is taken, from whichever module holds node-backup on its machine. +// backup is taken (`backed-up`), from whichever module holds node-backup on its machine (restic today). var NodeAgentHears = []string{"mesh.mod." + BackupHolderModule + ".event.backed-up"} // BackupHolderModule is the module that holds node-backup on every node, restic: the one whose word a node-engine hears. // A second holder of the seat is a change of this grant. const BackupHolderModule = "restic" -// ComposedDigestsConfig is the bucket of the compositions' digests as the controller asserts it: five revisions a key. -func ComposedDigestsConfig() jetstream.KeyValueConfig { - return jetstream.KeyValueConfig{ - Bucket: ControllerComposedBucket, - Description: "each composition's digest and the judgement of its move from what the node runs (novox/hq ADR 0539 " + - "clauses 2 and 14): written by the controller alone, read by the assigner; empty until release 6", - History: 5, - MaxValueSize: 4 << 10, - MaxBytes: 64 << 20, - Storage: jetstream.FileStorage, - } -} - -// EnsureComposedDigests asserts the bucket of the compositions' digests. -func EnsureComposedDigests(ctx context.Context, js jetstream.JetStream) error { - if _, err := js.CreateOrUpdateKeyValue(ctx, ComposedDigestsConfig()); err != nil { - return fmt.Errorf("asserting bucket %s: %w", ControllerComposedBucket, err) - } - return nil -} - // administers says a person may invoke every tool (`*`): the operator. func administers(invokes []string) bool { for _, i := range invokes { @@ -111,17 +138,17 @@ func administers(invokes []string) bool { // HandPathWrites is what the operator's account publishes for release 6's hand path (ADR 0539 clauses 2 and 10): a want // of any node, written and read back, in the assigner's bucket; each named node's compositions; the digests; and each -// node's object bucket, for a file over 64 KiB. +// node's object bucket for a file over 64 KiB, whose put reads an object's metadata and purges a replaced one's chunks. func HandPathWrites(nodes []string) ([]string, error) { out := append([]string{"$KV." + delivery.AssignerWantsBucket + ".want.>"}, readGrants(delivery.AssignerWantsBucket, "want.>")...) - out = append(out, "$KV."+ControllerComposedBucket+".>") - out = append(out, readGrants(ControllerComposedBucket, "")...) + out = append(out, "$KV."+delivery.ControllerComposedBucket+".>") + out = append(out, readGrants(delivery.ControllerComposedBucket, "")...) for _, node := range nodes { w, err := ComposedWrites(node) if err != nil { return nil, err } - bucket, _ := composedBucketOf(node) + bucket, _ := delivery.ComposedBucketFor(node) out = append(out, w) out = append(out, readGrants(bucket, "")...) files, err := DeclarationFilesWrites(node) @@ -129,9 +156,28 @@ func HandPathWrites(nodes []string) ([]string, error) { return nil, err } stream := "OBJ_" + DeclarationFilesBucket + node - // The object store's put reads an object's metadata already there and, replacing it, purges its old chunks. out = append(out, files, "$JS.API.STREAM.INFO."+stream, "$JS.API.DIRECT.GET."+stream+".>", "$JS.API.STREAM.MSG.GET."+stream, "$JS.API.STREAM.PURGE."+stream) } return out, nil } + +// DeclarationsStream is the stream of each node's last declaration, `mesh.node..declare` (issue 107). Release 6 +// sends none (ADR 0539 clauses 5 and 11): the stream is kept, with each node's consumer, only so a node away during the +// switch finds its last declaration and with it the new node-engine; the controller deletes it once every node's have +// names the new node-engine, and from then on neither asserts it nor any node's consumer on it. +const DeclarationsStream = "NODES" + +// RaiseAfterDeclarations is Raise once the declarations' stream is retired: the mesh's streams and consumers but the +// declarations' stream, and no node's consumer of a declaration. +func RaiseAfterDeclarations(r Raiser) error { + for _, s := range MeshStreams() { + if s.Name == DeclarationsStream { + continue + } + if err := r.EnsureStream(s); err != nil { + return fmt.Errorf("asserting stream %s: %w", s.Name, err) + } + } + return AssertMeshConsumers(r) +} diff --git a/internal/broker/composed_live_test.go b/internal/broker/composed_live_test.go index 5f5066ef..8c9cf985 100644 --- a/internal/broker/composed_live_test.go +++ b/internal/broker/composed_live_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -76,7 +77,7 @@ func TestNatsTheOperatorWritesTheHandPathOnTheComposedUserList(t *testing.T) { if err := EnsureDeclarationFiles(ctx, cjs, "ace"); err != nil { t.Fatal(err) } - if err := EnsureComposedDigests(ctx, cjs); err != nil { + if err := EnsureComposed(ctx, cjs); err != nil { t.Fatal(err) } if _, err := cjs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "composed-ace", History: 5}); err != nil { @@ -121,7 +122,7 @@ func TestNatsTheOperatorWritesTheHandPathOnTheComposedUserList(t *testing.T) { return err }) try("write its digest", func(c context.Context) error { - kv, err := js.KeyValue(c, ControllerComposedBucket) + kv, err := js.KeyValue(c, delivery.ControllerComposedBucket) if err != nil { return err } diff --git a/internal/broker/composed_test.go b/internal/broker/composed_test.go index 19e95265..56776e4f 100644 --- a/internal/broker/composed_test.go +++ b/internal/broker/composed_test.go @@ -5,15 +5,157 @@ import ( "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go/jetstream" ) +// 1945/AC1: a node's composition bucket keeps five revisions of a key and is named for its node; the bucket of digests +// likewise; a node name a subject would widen is refused. +func TestTheCompositionBuckets(t *testing.T) { + cfg, err := ComposedConfig("ace") + if err != nil || cfg.Bucket != "composed-ace" || cfg.History != 5 { + t.Fatalf("ComposedConfig(ace) = %+v, %v", cfg, err) + } + if d := ComposedDigestsConfig(); d.Bucket != delivery.ControllerComposedBucket || d.History != 5 { + t.Errorf("ComposedDigestsConfig = %+v", d) + } + if _, err := ComposedConfig("a.b"); err == nil { + t.Errorf("a dotted node got a bucket") + } +} + +// 1945/AC1: a node-engine reads its own compositions and its own wants and nothing of another node's, nor the digests; +// the controller writes a node's bucket by its own subjects. +func TestANodeReadsOnlyItsOwnCompositions(t *testing.T) { + reads, err := ComposedReads("ace") + if err != nil { + t.Fatal(err) + } + all := strings.Join(reads, " ") + for _, want := range []string{"KV_composed-ace", "$KV.mesh-delivery-assigner_wants.want.ace.>"} { + if !strings.Contains(all, want) { + t.Errorf("a node-engine of ace cannot reach %s: %v", want, reads) + } + } + for _, never := range []string{"composed-g14", delivery.ControllerComposedBucket, "$KV.composed-ace", "want.*", "want.>"} { + if strings.Contains(all, never) { + t.Errorf("a node-engine of ace is granted %s: %v", never, reads) + } + } + if w, err := ComposedWrites("ace"); err != nil || w != "$KV.composed-ace.>" { + t.Errorf("ComposedWrites(ace) = %q, %v", w, err) + } + if _, err := ComposedReads("a*"); err == nil { + t.Errorf("a node name with a wildcard was granted") + } +} + +// 1945 (for issue 1944): a node-engine asks its own machine's backup holder to back a data module up before it moves +// and when it last was, and no other machine's. +func TestANodeAgentAsksItsOwnBackupHolder(t *testing.T) { + asks, err := NodeAgentAsks("ace") + if err != nil || strings.Join(asks, " ") != "mesh.seat.node-backup.tool.now.ace mesh.seat.node-backup.tool.backed-up.ace" { + t.Errorf("NodeAgentAsks(ace) = %v, %v", asks, err) + } + p, err := PermissionsFor(Principal{Kind: KindNode, Node: "ace", PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + all := strings.Join(p.Publish, " ") + for _, want := range append(asks, "$JS.API.DIRECT.GET.KV_composed-ace.>") { + if !strings.Contains(all, want) { + t.Errorf("a node-engine of ace may not publish %s", want) + } + } + if !strings.Contains(strings.Join(p.Subscribe, " "), "mesh.mod.restic.event.backed-up") { + t.Errorf("a node-engine of ace does not hear a backup taken: %v", p.Subscribe) + } + if strings.Contains(all, "node-backup.tool.now.*") || strings.Contains(all, "KV_composed-g14") { + t.Errorf("a node-engine of ace reaches another machine: %v", p.Publish) + } +} + +// recordingRaiser records what a raise asserts. +type recordingRaiser struct{ streams, consumers []string } + +func (r *recordingRaiser) EnsureStream(s Stream) error { + r.streams = append(r.streams, s.Name) + return nil +} +func (r *recordingRaiser) EnsureConsumer(c Consumer) error { + r.consumers = append(r.consumers, c.Stream+"/"+c.Name) + return nil +} + +// 1945/AC14 (unit half): once the declarations' stream is retired, a raise asserts every stream of the mesh but it, +// and no node's consumer of a declaration; before, a raise asserts both (ADR 0539 clause 11). +func TestARaiseAfterTheDeclarationsAssertsNoneOfThem(t *testing.T) { + before := &recordingRaiser{} + if err := Raise(before, []string{"ace"}); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(before.streams, " "), DeclarationsStream) || !strings.Contains(strings.Join(before.consumers, " "), "NODES/ace") { + t.Fatalf("before the retirement a raise asserts %v and %v", before.streams, before.consumers) + } + after := &recordingRaiser{} + if err := RaiseAfterDeclarations(after); err != nil { + t.Fatal(err) + } + if strings.Contains(strings.Join(after.streams, " "), DeclarationsStream) || strings.Contains(strings.Join(after.consumers, " "), "NODES/") { + t.Errorf("after the retirement a raise asserts %v and %v", after.streams, after.consumers) + } + if len(after.streams) != len(before.streams)-1 { + t.Errorf("after the retirement %d streams are asserted, before %d", len(after.streams), len(before.streams)) + } +} + +// 1945 (the review of A1, (b) and H1): the operator's account, a person who may invoke every tool, writes release 6's +// hand path: a want of any node, each node's compositions, the digests, and a node's object bucket with what replacing +// an object needs; a person with fewer tools writes none of it. +func TestTheOperatorWritesTheHandPath(t *testing.T) { + users, err := Users(Records{Nodes: []string{"ace", "novox"}, People: map[string][]string{"jochen": {"*"}, "guest": {"mesh-controller.status"}}}) + if err != nil { + t.Fatal(err) + } + for _, u := range users { + if u.Kind != KindPerson { + continue + } + perms, err := PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{"$KV.mesh-delivery-assigner_wants.want.>", "$KV.composed-ace.>", "$KV.composed-novox.>", + "$KV.mesh-controller_composed.>", "$O.declaration-files-ace.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-ace.>", + "$JS.API.STREAM.MSG.GET.OBJ_declaration-files-ace", "$JS.API.STREAM.PURGE.OBJ_declaration-files-ace"} { + found := false + for _, p := range perms.Publish { + found = found || p == s + } + if found != (u.Module == "jochen") { + t.Errorf("%s may publish %s: %v", u.Module, s, found) + } + } + } + c, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + found := false + for _, p := range c.Publish { + found = found || p == "mesh.seat.mesh-controller.event.bucket-deleted" + } + if !found { + t.Errorf("the controller may not say bucket-deleted") + } +} + // 1945 (the review's A1): the bus's user list today's controller composes, which is the one the bus runs after release // 6's switch, already grants every subject the new controller, every new node-engine and the assigner use: the // controller writes each node's compositions and the digests; a node-engine reads its own compositions and wants, asks // its own backup holder and hears its word, and nothing of another node's; the assigner reads the digests, announced // readable. -func TestTheUserListBeforeTheSwitchGrantsWhatReleaseSixUses(t *testing.T) { +func TestTheUserListGrantsWhatReleaseSixUses(t *testing.T) { controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x", FilesFor: []string{"ace", "g14"}}) if err != nil { t.Fatal(err) @@ -42,7 +184,7 @@ func TestTheUserListBeforeTheSwitchGrantsWhatReleaseSixUses(t *testing.T) { } } if !slices.Contains(controllerReadableStates(), ControllerComposed) || - !slices.Contains(ControllerBuckets(), ControllerComposedBucket) || !IsControllerBucket(ControllerComposedBucket) { + !slices.Contains(ControllerBuckets(), delivery.ControllerComposedBucket) || !IsControllerBucket(delivery.ControllerComposedBucket) { t.Fatalf("the digests are not announced readable among the controller's buckets") } if !ControllerAnnouncement("novox").Readable(ControllerComposed) { @@ -59,30 +201,3 @@ func TestTheUserListBeforeTheSwitchGrantsWhatReleaseSixUses(t *testing.T) { t.Errorf("the digests' bucket is %+v", c) } } - -// 1945 (the review of A1, (b)): the operator's account, a person who may invoke every tool, writes release 6's hand -// path: a want of any node, each node's compositions and the digests; a person with fewer tools writes none of it. -func TestTheOperatorWritesTheHandPathAheadOfTheSwitch(t *testing.T) { - users, err := Users(Records{Nodes: []string{"ace", "novox"}, People: map[string][]string{"jochen": {"*"}, "guest": {"mesh-controller.status"}}}) - if err != nil { - t.Fatal(err) - } - for _, u := range users { - if u.Kind != KindPerson { - continue - } - perms, err := PermissionsFor(u) - if err != nil { - t.Fatal(err) - } - hand := []string{"$KV.mesh-delivery-assigner_wants.want.>", "$KV.composed-ace.>", "$KV.composed-novox.>", - "$KV.mesh-controller_composed.>", "$O.declaration-files-ace.>", - "$JS.API.DIRECT.GET.OBJ_declaration-files-ace.>", "$JS.API.STREAM.MSG.GET.OBJ_declaration-files-ace", - "$JS.API.STREAM.PURGE.OBJ_declaration-files-ace"} - for _, s := range hand { - if got := slices.Contains(perms.Publish, s); got != (u.Module == "jochen") { - t.Errorf("%s may publish %s: %v", u.Module, s, got) - } - } - } -} diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index f6f00782..13a2966a 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -39,17 +39,6 @@ var ( // each ask by its id, its options and the actions they stand for, how it ended and whether the // controller acted on its warrant — so a restart neither asks twice nor acts twice. AskedBucket = BucketName(ControllerSeat, "asked") - // MergeWindowBucket holds the clock of every open batch's merge window (novox/hq ADR 0407 decision 1): per - // batch a key `.quiet`, made again by each merge into it and living the window, and a key - // `.most`, made once when it opens and living the window's most. The first to expire closes the - // batch, and the controller's watcher hears the delete marker. Written by the lease holder alone. - MergeWindowBucket = BucketName(ControllerSeat, "merge-window") - // RolloutSilentBucket holds the wait of every machine a rollout has sent a tier to (novox/hq ADR 0406 §5, issue - // 974): per send a key `..` that lives ApplySilentAfter, made when the controller sends that - // machine and deleted by the machine's report on that send. Only a missing report lets it expire, and the - // controller's watcher hearing the expiry ends the wait and names the machine silent. Written by the lease - // holder alone. - RolloutSilentBucket = BucketName(ControllerSeat, "rollout-silent") // DeliveryMergesBucket holds the merges mesh-delivery announced it made, each with the ask whose Approve it merged // on (novox/hq ADR 0420 §7, issue 1025): per merge a key `///` holding the // announcement, kept DeliveryMergesKeptFor. What lets the build of a moved module register it from its new @@ -152,17 +141,11 @@ const ( MirrorMarkerTTL = 24 * time.Hour ) -// MergeWindowMarkerTTL is how long the marker an expired window key leaves is kept: long enough for a watcher -// that is up to hear it, short enough to leave nothing behind. -const MergeWindowMarkerTTL = time.Minute - -// RolloutSilentMarkerTTL is how long the marker an expired silent key leaves is kept, for the same reason. -const RolloutSilentMarkerTTL = time.Minute - -// RolloutSilentKeptFor is the age of the silent keys' bucket: every report deletes a key and the delete leaves a marker, which -// with one value per key would stay for ever and fill the bucket. A key lives a quarter of an hour at most, so an hour -// takes nothing live. -const RolloutSilentKeptFor = time.Hour +// PlanStateBuckets are the buckets of the plans' state that release 6 deletes (novox/hq ADR 0539 clause 13): the merge +// window's clock, the silent sends' waits and the building plans' hold. The controller deletes each at its first start +// and says it once (composed_start.go); none is asserted, granted or written any more. +var PlanStateBuckets = []string{BucketName(ControllerSeat, "merge-window"), BucketName(ControllerSeat, "rollout-silent"), + BucketName(ControllerSeat, "plans-building")} // AskedKeptFor is how long an ask is kept after it was made: a month, as the router keeps its own. const AskedKeptFor = 30 * 24 * time.Hour @@ -193,18 +176,17 @@ func IsControllerBucket(bucket string) bool { bucket == ConditionHistoryBucket || bucket == LeaseBucket || bucket == AskedBucket || bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket || bucket == DeclaredPackagesBucket || - bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket || bucket == delivery.HaveBucket || bucket == StatesAnnouncedBucket || - bucket == GrantsBucket || bucket == ControllerComposedBucket + bucket == GrantsBucket || bucket == delivery.ControllerComposedBucket } // ControllerBuckets are the controller's own buckets, in the order they are asserted. func ControllerBuckets() []string { return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket, - AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, DeclaredPackagesBucket, MergeWindowBucket, RolloutSilentBucket, + AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, DeclaredPackagesBucket, DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket, StatesAnnouncedBucket, GrantsBucket, - ControllerComposedBucket} + delivery.ControllerComposedBucket} } // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection. @@ -301,40 +283,9 @@ func (j *JetStream) EnsureControllerBuckets() error { }); err != nil { return fmt.Errorf("asserting bucket %s: %w", AskedBucket, err) } - // **The merge window: keys that expire.** A key's own time to live is the window; the marker an expired key - // leaves is what a watcher hears, kept a minute (ADR 0407 decision 1). - if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: MergeWindowBucket, - Description: "the clock of every open batch's merge window (novox/hq ADR 0407): `.quiet` lives the " + - "window and is made again by each merge, `.most` lives the most; the first to expire closes the " + - "batch; written by the controller alone", - History: 1, - MaxValueSize: 4 << 10, - MaxBytes: 8 << 20, - Storage: jetstream.FileStorage, - LimitMarkerTTL: MergeWindowMarkerTTL, - }); err != nil { - return fmt.Errorf("asserting bucket %s: %w", MergeWindowBucket, err) - } - // **The rollout's silent keys: keys that expire** (ADR 0406 §5, issue 974). A key lives as long as the machine - // may stay silent; the machine's report deletes it, and a marker of expiry is what the watcher hears. - if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ - Bucket: RolloutSilentBucket, - Description: "the wait of every machine a rollout sent a tier to (novox/hq ADR 0406): `..` " + - "lives the time a machine may stay silent and is deleted by its report; one that expires names the " + - "machine silent; written by the controller alone", - History: 1, - TTL: RolloutSilentKeptFor, - MaxValueSize: 4 << 10, - MaxBytes: 8 << 20, - Storage: jetstream.FileStorage, - LimitMarkerTTL: RolloutSilentMarkerTTL, - }); err != nil { - return fmt.Errorf("asserting bucket %s: %w", RolloutSilentBucket, err) - } - // **The compositions' digests** (ADR 0539 clauses 2 and 6), asserted ahead of release 6: what its assigner reads. - if err := EnsureComposedDigests(ctx, js); err != nil { - return err + // **The compositions' digests** (ADR 0539 clauses 2 and 6): what the assigner reads before it writes a want. + if _, err := js.CreateOrUpdateKeyValue(ctx, ComposedDigestsConfig()); err != nil { + return fmt.Errorf("asserting bucket %s: %w", delivery.ControllerComposedBucket, err) } // **The states each module announced** (novox/hq ADR 0495 clauses 2 and 3, issue 1790): one key a module, its last word. if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ diff --git a/internal/broker/controller_buckets_integration_test.go b/internal/broker/controller_buckets_integration_test.go index 9f6e8d5d..bf8a1ca4 100644 --- a/internal/broker/controller_buckets_integration_test.go +++ b/internal/broker/controller_buckets_integration_test.go @@ -4,7 +4,6 @@ package broker import ( "context" - "slices" "testing" "time" @@ -42,152 +41,6 @@ func TestWhatTheControllerAskedIsBounded(t *testing.T) { } } -func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) { - controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(controller.Publish, "$KV."+MergeWindowBucket+".>") { - t.Error("the controller may not write the merge window's bucket") - } - if !IsControllerBucket(MergeWindowBucket) || !slices.Contains(ControllerBuckets(), MergeWindowBucket) { - t.Error("the merge window's bucket is not asserted as the controller's own") - } - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - kv, err := api.KeyValue(ctx, MergeWindowBucket) - if err != nil { - t.Fatal(err) - } - w, err := kv.WatchAll(ctx) - if err != nil { - t.Fatal(err) - } - defer w.Stop() - if _, err := kv.Create(ctx, "plan-1.quiet", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil { - t.Fatal(err) - } - for { - select { - case e := <-w.Updates(): - if e != nil && e.Key() == "plan-1.quiet" && e.Operation() != jetstream.KeyValuePut { - return - } - case <-ctx.Done(): - t.Fatal("no watcher heard the key expire") - } - } -} - -func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) { - controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") { - t.Error("the controller may not write the rollout's silent keys' bucket") - } - if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) { - t.Error("the rollout's silent keys' bucket is not asserted as the controller's own") - } - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - kv, err := api.KeyValue(ctx, RolloutSilentBucket) - if err != nil { - t.Fatal(err) - } - w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) - if err != nil { - t.Fatal(err) - } - defer w.Stop() - if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil { - t.Fatal(err) - } - if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil { - t.Fatal(err) - } - if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil { - t.Fatal(err) - } - for { - select { - case e := <-w.Updates(): - if e == nil || e.Operation() == jetstream.KeyValuePut { - continue - } - if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete { - t.Fatalf("a report's delete was heard as %v", e.Operation()) - } - if e.Key() == "plan-1.0.g14" { - if e.Operation() != jetstream.KeyValuePurge { - t.Fatalf("an expiry was heard as %v, not a purge", e.Operation()) - } - return - } - case <-ctx.Done(): - t.Fatal("no watcher heard the key expire") - } - } -} - -func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) { - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - // As the first build made it: no age. - if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1, - MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatalf("a bucket made without the age is not brought to it: %v", err) - } - kv, err := api.KeyValue(ctx, RolloutSilentBucket) - if err != nil { - t.Fatal(err) - } - status, err := kv.Status(ctx) - if err != nil { - t.Fatal(err) - } - if status.TTL() != RolloutSilentKeptFor { - t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor) - } -} - func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testing.T) { js, err := Dial(testbus.URL(t)) if err != nil { diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index 0c7441cd..a492d856 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -116,7 +116,7 @@ func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) { rowed := map[string]bool{} for _, row := range WritersTable { if !strings.HasSuffix(row.State, " mirror") && !strings.HasSuffix(row.State, " of the chain") && row.State != "the grants issued" && - row.State != "the compositions' digests" { + row.State != "the compositions and their digests" { continue } for _, subject := range row.Subjects { diff --git a/internal/broker/genesis_template_test.go b/internal/broker/genesis_template_test.go index bcf1cf1f..ce8a0b20 100644 --- a/internal/broker/genesis_template_test.go +++ b/internal/broker/genesis_template_test.go @@ -49,7 +49,9 @@ func TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose(t *testing.T) // builder (issue 1152): the controller derives none of them (R4g, ADR 0525 §10), the first push takes them away. carriedPub = slices.DeleteFunc(slices.Clone(carriedPub), func(s string) bool { return strings.HasPrefix(s, "mesh.seat.node-build-agent.") || strings.HasPrefix(s, "mesh.seat.mesh-build-machine.") || - strings.HasPrefix(s, "$KV.SEAT_NODE_BUILD_AGENT_") || strings.HasPrefix(s, "$KV.SEAT_MESH_BUILD_MACHINE_") + strings.HasPrefix(s, "$KV.SEAT_NODE_BUILD_AGENT_") || strings.HasPrefix(s, "$KV.SEAT_MESH_BUILD_MACHINE_") || + // And the plans' events release 6 retired (novox/hq ADR 0539), until mesh-host's genesis template drops them. + s == "mesh.seat.mesh-controller.event.plan-moved" || s == "mesh.seat.mesh-controller.event.sent" }) if diff := missing(carriedPub, want.Publish); len(diff) > 0 { t.Errorf("the installer's user list lets the controller publish %v, which it does not derive", diff) diff --git a/internal/broker/log_test.go b/internal/broker/log_test.go index 1d8ac164..c41038c3 100644 --- a/internal/broker/log_test.go +++ b/internal/broker/log_test.go @@ -171,8 +171,8 @@ func TestRaisingLogsReportsWhatNothingDeclares(t *testing.T) { // ADR 0201 §15–16): no code of this repository outside its tests calls the client's stream or bucket // delete, or purges a stream, but for the controller lease's own stream, which purges its own history // below a sequence (internal/lease), and the controller's record mirrors, which purge the key of a record -// the store no longer holds (internal/recordmirror; novox/hq ADR 0289, design 54 §2). A new call is a decision, -// not a refactor. +// the store no longer holds (internal/recordmirror; novox/hq ADR 0289, design 54 §2), and the buckets of the plans' +// state the controller deletes once at its first start (ADR 0539 clause 13). A new call is a decision, not a refactor. func TestNoPathDeletesALogOrABucket(t *testing.T) { removers := map[string]bool{"DeleteStream": true, "DeleteKeyValue": true, "PurgeStream": true, "DeleteObjectStore": true} @@ -199,15 +199,19 @@ func TestNoPathDeletesALogOrABucket(t *testing.T) { name := sel.Sel.Name decided := strings.Contains(filepath.ToSlash(path), "internal/lease/") || strings.Contains(filepath.ToSlash(path), "internal/recordmirror/") - if removers[name] || (name == "Purge" && !decided) { + // And the buckets of the plans' state release 6 deleted, at the controller's first start, each said (novox/hq + // ADR 0539 clause 13): the one decided deletion of a bucket. + release6 := strings.HasSuffix(filepath.ToSlash(path), "cmd/mesh-controller/composed_start.go") && + (name == "DeleteKeyValue" || name == "DeleteStream") // and the declarations' stream once no node reads one (clause 11) + if (removers[name] && !release6) || (name == "Purge" && !decided) { found = append(found, path+": "+name) } return true }) for _, lit := range stringsIn(f) { if strings.Contains(lit, "$JS.API.STREAM.DELETE") || strings.Contains(lit, "$JS.API.STREAM.PURGE") { - // Only the writers table names it, as a subject no one but the controller may publish; and the operator's - // release 6 hand path, whose object store purges the chunks of an object it replaces in a node's + // Only the writers table names it, as a subject no one but the controller may publish. + // And the operator's release 6 hand path, whose object store purges a replaced object's chunks in a node's // object bucket (novox/hq ADR 0539 clause 10): a grant, never a call of this repository. if !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/writers.go") && !strings.HasSuffix(filepath.ToSlash(path), "internal/broker/composed.go") { diff --git a/internal/broker/nats.go b/internal/broker/nats.go index f605f6f5..9b93db72 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -396,11 +396,6 @@ var DegradableGrants = []DegradableGrant{ Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at " + "its first push, which is when it acknowledges a merge (ADR 0407 decision 3, issue 973)", }, - { - Subject: "$KV." + MergeWindowBucket + ".>", - Reason: "refused: the controller closes a batch's window by a timer of its own for that batch, as the " + - "window setting says (ADR 0407 decision 1, issue 973)", - }, { Subject: "$KV." + DeclaredPackagesBucket + ".>", Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + @@ -429,13 +424,13 @@ var DegradableGrants = []DegradableGrant{ }, { Subject: "mesh.seat." + ControllerSeat + ".event.bucket-deleted", - Reason: "refused: nothing is deleted before release 6, whose controller composes its own permissions into the bus " + - "module's composition before its first start's deletions (novox/hq ADR 0539 clause 13)", + Reason: "refused: nothing is deleted at genesis; the controller composes its own permissions into the bus module's " + + "composition before a later start's deletions (novox/hq ADR 0539 clause 13)", }, { - Subject: "$KV." + ControllerComposedBucket + ".>", - Reason: "carried only at genesis: nothing writes a composition's digest before release 6, whose controller " + - "composes its own permissions into the bus module's composition (novox/hq ADR 0539 clauses 2 and 7)", + Subject: "$KV." + delivery.ControllerComposedBucket + ".>", + Reason: "carried only at genesis: the controller composes its own permissions into the bus module's composition, " + + "and writes a composition's digest only after the bus applied it (novox/hq ADR 0539 clauses 2 and 7)", }, { Subject: "$KV." + GrantsBucket + ".>", @@ -448,11 +443,6 @@ var DegradableGrants = []DegradableGrant{ Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + "first push, and writes the chain only after that, holding the lease (ADR 0408 §2 and §8, issue 1089)", }, - { - Subject: "$KV." + RolloutSilentBucket + ".>", - Reason: "refused: the controller looks at the plans when a wait for a machine's report is up, by a timer of its own " + - "for that wait (ADR 0406 §5, issue 974)", - }, } // perMachineEvents are a node-scoped seat's events about the holder itself, whose last token is the @@ -748,7 +738,7 @@ func PermissionsFor(p Principal) (Permissions, error) { return Permissions{}, err } pub = append(pub, writes) - // And each node's compositions, which it alone writes from release 6 on (novox/hq ADR 0539 clause 2). + // And each node's compositions, which it alone writes (novox/hq ADR 0539 clause 2). composed, err := ComposedWrites(node) if err != nil { return Permissions{}, err @@ -790,8 +780,7 @@ func PermissionsFor(p Principal) (Permissions, error) { // itself, its replies to the asker's own inbox. pub = append(pub, discovering()...) // And, for an administrator (`*`, the operator), release 6's hand path (ADR 0539 clauses 2 and 10): a want written - // at the terminal, and a composition made by hand with mesh-compose. Granted ahead of the switch, since the way back - // right after it runs on the user list today's controller composes. + // at the terminal, and a composition made by hand with mesh-hand-compose. if administers(p.Invokes) { hand, err := HandPathWrites(p.FilesFor) if err != nil { @@ -871,18 +860,21 @@ func PermissionsFor(p Principal) (Permissions, error) { return Permissions{}, err } pub = append(pub, fileReads...) - // And, from release 6 on, its own compositions and wants, its own backup holder's verbs and word (novox/hq ADR - // 0539 clauses 2, 9 and 16): granted ahead, since today's controller composes the user list after the switch. + // And its own compositions and wants, which it follows (novox/hq ADR 0539 clauses 1, 2 and 9), and nothing of another + // node's. composedReads, err := ComposedReads(p.Node) if err != nil { return Permissions{}, err } pub = append(pub, composedReads...) + // And its own machine's backup holder, before a data module moves (ADR 0539 clause 16). backups, err := NodeAgentAsks(p.Node) if err != nil { return Permissions{}, err } pub = append(pub, backups...) + // And the backup holder's word that it backed the module up (ADR 0539 clause 16), whichever module holds + // node-backup: one event, by its name. sub = append(sub, NodeAgentHears...) // And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the // values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call. diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index 33479446..bc2695c2 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -243,6 +243,7 @@ func TestAnAdministratorMayAskAnyTool(t *testing.T) { func TestAPersonReachesNothingButTools(t *testing.T) { perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"}) + // The one exception: the operator's release 6 hand path (novox/hq ADR 0539 clauses 2 and 10). hand, _ := HandPathWrites(nil) for _, p := range perms.Publish { if slices.Contains(hand, p) { diff --git a/internal/broker/streams.go b/internal/broker/streams.go index 87cb916f..052d47a8 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -373,17 +373,14 @@ var ControllerStates = []string{"applied", "refused", "built-before", // And a pull request's merge check, judged (novox/hq to-be 45 §9): what the forge's holder sets as // the pull request's status, and anybody else may read. "checked", - // And a walk kept in a new state (novox/hq ADR 0239): what the delivery it walks reads its steps from. - "plan-moved", - // And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974). - "sent", + // (A walk kept in a new state and a tier sent to a machine are gone with the plans, novox/hq ADR 0539.) // And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and // each registered module's declared config (design 57 F3). "assignment-changed", "declared-config-changed", "predefined-changed", // And a builder's key not taken in because the artifact store does not hold what it names (novox/hq issue 1937). "artifacts-gone", - // And, from release 6 on, a bucket of the plans' state deleted at the new controller's first start, and the - // declarations' stream once no node reads one (novox/hq ADR 0539 clauses 11 and 13): granted ahead of the switch. + // And a bucket of the plans' state deleted at the controller's first start, and the declarations' stream once no node + // reads one (novox/hq ADR 0539 clauses 11 and 13). "bucket-deleted"} // BusAdvisories are what the bus server says about the mesh's own account that the controller diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 52e6ed88..b64cf1f5 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -29,7 +29,7 @@ accounts { ] users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_composed.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.bucket-deleted", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_composed.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.bucket-deleted", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/broker/users.go b/internal/broker/users.go index 3f30f312..482207df 100644 --- a/internal/broker/users.go +++ b/internal/broker/users.go @@ -232,7 +232,7 @@ func Users(r Records) ([]Principal, error) { for _, person := range sortedNames(r.People) { p := Principal{Kind: KindPerson, Module: person, Invokes: r.People[person]} if administers(p.Invokes) { - // The operator's hand path from release 6 on writes each node's compositions (ADR 0539 clause 10). + // The operator's hand path writes each node's compositions (ADR 0539 clause 10). p.FilesFor = sortedCopy(r.Nodes) } out = append(out, p) diff --git a/internal/broker/writers.go b/internal/broker/writers.go index 7894f5fe..31437f66 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -142,21 +142,16 @@ var WritersTable = []WriterRow{ Subjects: []string{"mesh.control.*.report", "mesh.control.*.health", "mesh.control.*.cli"}, Writes: ownMachine}, {State: "the controller lease", Writer: "the controller instance holding it", KeptIn: "key-value " + LeaseBucket, Others: "a candidate waits", Subjects: kvOf(LeaseBucket), Writes: isController}, - // And the clock of an open batch's merge window, a plan before it is one (novox/hq ADR 0407 decision 1), and the - // wait of a rollout for each machine it sent (ADR 0406 §5, issue 974): keys that expire in buckets of their own, - // written by the lease holder. - {State: "plans and their tiers", Writer: "controller (lease holder), compare-and-set on the plan's revision", - KeptIn: "the controller's store; a batch's merge window in key-value " + MergeWindowBucket + - ", a rollout's wait for a machine's report in key-value " + RolloutSilentBucket + - ", and the merges mesh-delivery announced, for a moved module's registration (ADR 0420 §7), in key-value " + - DeliveryMergesBucket, Others: "read through plans", - Subjects: append(append(kvOf(MergeWindowBucket), kvOf(RolloutSilentBucket)...), kvOf(DeliveryMergesBucket)...), - Writes: isController}, - // The compositions' digests, granted and asserted ahead of release 6, which writes them (ADR 0539 clauses 2 and 6). - {State: "the compositions' digests", Writer: "controller (lease holder), compare-and-set, from release 6; the operator's " + - "account on the hand path alone (ADR 0539 clauses 2 and 10)", - KeptIn: "key-value " + ControllerComposedBucket, Others: "the assigner reads them", - Subjects: kvOf(ControllerComposedBucket), Writes: isControllerOrTheHandPath}, + // The merges mesh-delivery announced, for a moved module's registration (ADR 0420 §7), written by the lease holder. + {State: "the merges mesh-delivery announced", Writer: "controller (lease holder)", + KeptIn: "key-value " + DeliveryMergesBucket, Others: "read at a moved module's registration", + Subjects: kvOf(DeliveryMergesBucket), Writes: isController}, + // And the compositions (novox/hq ADR 0539 clauses 2 and 4): each node's in its own bucket, which only that node's + // node-engine reads, and each digest with the judgement of its move, which the assigner reads. + {State: "the compositions and their digests", Writer: "controller (lease holder), compare-and-set", + KeptIn: "key-value composed- for each node, and " + delivery.ControllerComposedBucket, + Others: "a node-engine reads its own node's; the assigner reads the digests", + Subjects: kvOf(delivery.ControllerComposedBucket), Writes: isControllerOrTheHandPath}, {State: "conditions", Writer: "controller", KeptIn: "key-value " + ConditionsBucket + " (and its history, " + ConditionHistoryBucket + "); how often each was observed, when last and its newest evidence in the " + "controller's store, so a look that changes nothing writes no bucket", Others: "raise or clear only through observations the controller reads", diff --git a/internal/broker/writers_test.go b/internal/broker/writers_test.go index 9614f3ba..2860ed10 100644 --- a/internal/broker/writers_test.go +++ b/internal/broker/writers_test.go @@ -20,9 +20,10 @@ var designRows = []string{ "a judgement of a machine's directories asked of it", "a machine's applied state and its report", "the controller lease", - "plans and their tiers", - // Release 6's compositions' digests, granted ahead of it (ADR 0539 clauses 2 and 6, issue 1945). - "the compositions' digests", + // The merges mesh-delivery announced, once part of the plans' row (ADR 0420 §7), and the compositions that replace + // the plans (ADR 0539 clauses 2 and 4). + "the merges mesh-delivery announced", + "the compositions and their digests", "conditions", "calls and their outcomes", "the hand-act log", diff --git a/internal/chain/chain_test.go b/internal/chain/chain_test.go index 42c216a0..68698bbc 100644 --- a/internal/chain/chain_test.go +++ b/internal/chain/chain_test.go @@ -49,23 +49,23 @@ func TestABuildIsKeptWholeAndReadBackWithItsReuseKeyAndWhatItPublished(t *testin func TestAWriteAgainstAMovedRevisionIsRefusedAndNothingIsLaidOver(t *testing.T) { ctx := context.Background() c := InMemory(nil) - first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) + first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}) if err != nil { t.Fatal(err) } if _, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes"}); !errors.Is(err, ErrExists) { t.Fatalf("a create of a key that holds a value answered %v", err) } - second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c2"}, first) + second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c2"}, first) if err != nil { t.Fatal(err) } // A writer that read the first revision writes now. - if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "stale"}, first); !errors.Is(err, ErrMoved) { + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "stale"}, first); !errors.Is(err, ErrMoved) { t.Fatalf("a write at a moved revision answered %v", err) } got, _, _ := c.Assignments.Get(ctx, "anchor.notes") - if got.Value.Commit != "c2" || got.Revision != second { + if got.Value.RequiredBy != "c2" || got.Revision != second { t.Fatalf("the key that moved was laid over: %+v at %d", got.Value, got.Revision) } } @@ -91,8 +91,8 @@ func TestNoKeyOfTheChainIsEverRemoved(t *testing.T) { } ctx := context.Background() c := InMemory(nil) - rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) - if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}, rev); err != nil { + rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}) + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}, rev); err != nil { t.Fatal(err) } keys, err := c.Assignments.Keys(ctx) @@ -190,31 +190,36 @@ func TestARolloutNamesEveryAssignmentItChangedWithBuildBeforeAndAfter(t *testing } } -// **An assignment names its build, artifact or package, commit and rollout**, and keeps the fields its readers read -// under the names they read them by (ADR 0408 §3). -func TestAnAssignmentNamesBuildArtifactCommitAndRollout(t *testing.T) { +// **An assignment is placement only** (ADR 0539 §12): the node, the module, since when, what requires it and its +// provisioning plan; no build, artifact, package, commit, rollout or time since is written in it, and a value of before +// release 6 that names them is read with them ignored. +func TestAnAssignmentIsPlacementOnly(t *testing.T) { a := Assignment{Node: "anchor", Module: "notes", Assigned: at(3), RequiredBy: "wiki", - Provisioning: json.RawMessage(`{"plan":1}`), Build: "b2", Artifact: "sha256:bb", Commit: "c2", Rollout: "r1", Since: at(4)} + Provisioning: json.RawMessage(`{"plan":1}`)} raw, _ := json.Marshal(a) for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"assigned":"2026-10-18T03:00:00Z"`, - `"required-by":"wiki"`, `"provisioning":{"plan":1}`, `"build":"b2"`, `"artifact":"sha256:bb"`, `"commit":"c2"`, - `"rollout":"r1"`, `"since":"2026-10-18T04:00:00Z"`} { + `"required-by":"wiki"`, `"provisioning":{"plan":1}`} { if !strings.Contains(string(raw), want) { t.Errorf("the assignment's value lacks %s: %s", want, raw) } } + for _, not := range []string{`"build"`, `"artifact"`, `"package"`, `"commit"`, `"rollout"`, `"since"`, `"pulled-in"`, `"sent-unknown"`} { + if strings.Contains(string(raw), not) { + t.Errorf("the assignment's value names %s: %s", not, raw) + } + } if a.Key() != "anchor.notes" { t.Errorf("key %s", a.Key()) } - // An assignment of today carries none of the new fields, and says none: a reader of today reads the bytes it did. - raw, _ = json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at(3)}) - if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` { - t.Errorf("an assignment without a chain is %s", raw) + // A value written before release 6 is read, its chain fields ignored, and written back without them. + var old Assignment + if err := json.Unmarshal([]byte(`{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z","build":"b2","commit":"c2",`+ + `"rollout":"r1","since":"2026-10-18T04:00:00Z","pulled-in":true}`), &old); err != nil { + t.Fatal(err) } - // A package in place of an artifact. - raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Package: &Package{Name: "n", Version: "1"}}) - if !strings.Contains(string(raw), `"package":{"name":"n","version":"1"}`) { - t.Errorf("the package is %s", raw) + raw, _ = json.Marshal(old) + if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` { + t.Errorf("an old value written again is %s", raw) } } @@ -341,7 +346,7 @@ func TestAnEmptyBucketIsReadAsEmpty(t *testing.T) { func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) { ctx := context.Background() c := InMemory(nil) - rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), Commit: "c1"}) + rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), RequiredBy: "c1"}) if err != nil { t.Fatal(err) } @@ -355,7 +360,7 @@ func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) { t.Fatal(err) } got, found, _ := c.Assignments.Get(ctx, "anchor.notes") - if !found || got.Value.Stands() || got.Value.Commit != "c1" || got.Value.Ended.IsZero() { + if !found || got.Value.Stands() || got.Value.RequiredBy != "c1" || got.Value.Ended.IsZero() { t.Fatalf("the ended value is %+v (found %v)", got.Value, found) } if history, _ := c.Assignments.History(ctx, "anchor.notes"); len(history) != 2 || !history[0].Value.Stands() { diff --git a/internal/chain/pending_test.go b/internal/chain/pending_test.go index 655aae40..c0e73f9d 100644 --- a/internal/chain/pending_test.go +++ b/internal/chain/pending_test.go @@ -61,27 +61,3 @@ func TestTheTransitionsOfAPendingAssignment(t *testing.T) { t.Error("a claim moves a wait to applying and back") } } - -// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit -// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have -// none; and unknown is never written in `commit`. -func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) { - at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at} - if !pulled.Stands() || pulled.State != "" { - t.Fatalf("a pulled-in value does not stand: %+v", pulled) - } - raw, _ := json.Marshal(pulled) - if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) { - t.Fatalf("the pulled-in value is %s", raw) - } - unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at} - raw, _ = json.Marshal(unknown) - if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() { - t.Fatalf("the hand-sent value is %s", raw) - } - plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at}) - if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") { - t.Fatalf("a value that is neither says so: %s", plain) - } -} diff --git a/internal/chain/records.go b/internal/chain/records.go index 2738abd2..dcb33282 100644 --- a/internal/chain/records.go +++ b/internal/chain/records.go @@ -175,9 +175,10 @@ type Rollout struct { Links } -// Assignment is one module on one node, kept under `.`. The first five fields are the ones its readers -// have always read, with the names they have always had (novox/hq ADR 0408 §3); the rest name the chain: the build, the -// artifact (by digest) or the package it runs, its commit, the rollout that put it there and since when. +// Assignment is one module on one node, kept under `.`: placement only (novox/hq ADR 0539 §12). The first +// five fields are the ones its readers have always read, with the names they have always had (novox/hq ADR 0408 §3); +// no build, commit or rollout is written in it any more, and a value written before release 6 that names them is read +// with them ignored. type Assignment struct { Node string `json:"node"` Module string `json:"module"` @@ -185,13 +186,6 @@ type Assignment struct { RequiredBy string `json:"required-by,omitempty"` Provisioning json.RawMessage `json:"provisioning,omitempty"` - Build string `json:"build,omitempty"` - Artifact string `json:"artifact,omitempty"` - Package *Package `json:"package,omitempty"` - Commit string `json:"commit,omitempty"` - Rollout string `json:"rollout,omitempty"` - Since time.Time `json:"since,omitzero"` - // Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed** // (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a // State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that @@ -201,14 +195,6 @@ type Assignment struct { // Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when // it is made, carried into the value that stands: the record of what became of the wait. Pending *Pending `json:"pending,omitempty"` - - // PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a - // send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read - // (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine. - PulledIn bool `json:"pulled-in,omitempty"` - // SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the - // commit is cleared with it. Absent normally; a send that names its builds clears it. - SentUnknown time.Time `json:"sent-unknown,omitzero"` } // StatePending is the State of a value that holds no assignment but a pending one. diff --git a/internal/inventory/batches.go b/internal/inventory/batches.go deleted file mode 100644 index 8d5cf20e..00000000 --- a/internal/inventory/batches.go +++ /dev/null @@ -1,138 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "strings" - "time" -) - -// The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a -// module follows is kept here once, with the batch or walk that answers it. A merge is never dropped and never -// walked twice: its key is its repository and commit, and it belongs to one plan at a time. - -// BatchedMerge is one merge as the controller heard it. -type BatchedMerge struct { - Repository string - Branch string - Commit string - // Merged is when the forge made it: the branch's order. Heard is when the controller heard it: what the - // window is measured from. - Merged time.Time - Heard time.Time - // Event is the forge's announcement, what the cut plans from. - Event json.RawMessage - // Plan is the batch or walk answering it; empty while it waits to be walked alone. - Plan string - // Alone says it is walked on its own commit, after a failed walk that carried it in a later one. - Alone bool -} - -// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan. -func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return false, err - } - return i.chainAddMerge(ctx, c, m) -} - -// MergeOf is one kept merge; false when it was never heard. -func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return BatchedMerge{}, false, err - } - return i.chainMergeOf(ctx, c, repository, commit) -} - -// MergesOf is every merge a plan answers, oldest merge first. -func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainMergesOf(ctx, c, plan) -} - -// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the -// newest first. -func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainLaterMergesOf(ctx, c, repository, branch, after) -} - -// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a -// module's last look says (novox/hq ADR 0276). -func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) { - c, err := i.theChain(ctx) - if err != nil { - return false, time.Time{}, err - } - return i.chainKeptOnBranch(ctx, c, repository, branch) -} - -// EarliestKept says the controller keeps a merge of some repository's branch, and when it heard the first of all: where its record of -// merges begins on the whole mesh (novox/hq ADR 0463). A merge made before then is history on every trunk. -func (i *Inventory) EarliestKept(ctx context.Context) (bool, time.Time, error) { - c, err := i.theChain(ctx) - if err != nil { - return false, time.Time{}, err - } - return i.chainEarliestKept(ctx, c) -} - -// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3). -func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainAloneMerges(ctx, c) -} - -// WaitingMerges is every merge no plan answers that is not walked alone: kept, and waiting for something else to -// happen to it (a first merge into a repository, waiting for mesh-delivery's word on it, issue 1146). -func (i *Inventory) WaitingMerges(ctx context.Context) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainWaitingMerges(ctx, c) -} - -// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to -// be walked on its own commit. -func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error { - c, err := i.theChain(ctx) - if err != nil { - return err - } - return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}}) -} - -// HoldMerge puts a merge back to waiting, no plan answering it, with the event it is kept under replaced (novox/hq issue 1173: -// a first merge held for a build-agent that reads its manifest, the hold said in its event). -func (i *Inventory) HoldMerge(ctx context.Context, repository, commit string, event json.RawMessage) error { - c, err := i.theChain(ctx) - if err != nil { - return err - } - return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Event: event}}) -} - -// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more -// are read so a fault is seen. -func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated) -} - -// SameRepository says two spellings of a repository are one. -func SameRepository(a, b string) bool { return strings.EqualFold(a, b) } diff --git a/internal/inventory/build_lookups.go b/internal/inventory/build_lookups.go new file mode 100644 index 00000000..28647c48 --- /dev/null +++ b/internal/inventory/build_lookups.go @@ -0,0 +1,26 @@ +package inventory + +import "context" + +// Lookups of builds the delivery plans once held, which other code still reads. + +// BuildOf is the id of the newest successful build of a module from a commit; empty when none is +// recorded (a manifest handed over by hand). +func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { + c, err := i.theChain(ctx) + if err != nil { + return "", err + } + return i.chainBuildOf(ctx, c, module, commit) +} + +// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a +// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told +// from one that does (the bus's planned step, ADR 0236). +func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { + c, err := i.theChain(ctx) + if err != nil { + return nil, err + } + return i.chainBuildFingerprints(ctx, c, module) +} diff --git a/internal/inventory/busy_integration_test.go b/internal/inventory/busy_integration_test.go deleted file mode 100644 index 61283905..00000000 --- a/internal/inventory/busy_integration_test.go +++ /dev/null @@ -1,66 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, - Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}} - keep := func(busy []BusyWait) bool { - t.Helper() - news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1, - Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false }) - if err != nil { - t.Fatal(err) - } - return news - } - read := func() []BusyWait { - t.Helper() - reports, err := inv.LastReports(ctx) - if err != nil { - t.Fatal(err) - } - for _, r := range reports { - if r.Node == node { - return r.Busy - } - } - t.Fatal("the machine's report is not read back") - return nil - } - keep(nil) - if got := read(); len(got) != 0 { - t.Fatalf("a report with nothing busy reads back %+v", got) - } - if !keep(waiting) { - t.Error("a container starting to wait is not news") - } - got := read() - if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" { - t.Fatalf("the wait is not read back: %+v", got) - } - again := []BusyWait{waiting[0]} - again[0].Said = "busy: a check, a minute on" - if keep(again) { - t.Error("a wait whose output changed is news") - } - if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle", - Since: since, Until: since.Add(15 * time.Minute)}}) { - t.Error("a container replaced after it waited is not news") - } - if _, ok := WaitingFor(read(), "build-agent"); ok { - t.Error("a container replaced still reads as waiting") - } -} diff --git a/internal/inventory/busy_test.go b/internal/inventory/busy_test.go deleted file mode 100644 index 7170119d..00000000 --- a/internal/inventory/busy_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package inventory - -import ( - "strings" - "testing" - "time" -) - -// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the -// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the -// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said -// the container replaced. -func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) { - raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`, - `"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z", - "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery", - "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1) - w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - if w.TotalMS != 282894 { - t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS) - } - busy := phase(w, PhaseBusy, 0) - if busy.State != PhaseMeasured || busy.TookMS != 60000 { - t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy) - } - if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 { - t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged) - } - var names []string - for _, ph := range w.Phases { - names = append(names, ph.Name) - } - if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) { - t.Fatalf("the busy wait is not between the send and the judging: %v", names) - } -} - -// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no -// such phase. -func TestAnOpenBusyWaitIsSaid(t *testing.T) { - raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3, - "verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`, - `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", - "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery", - "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1) - raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1) - p := walkOf(t, raw) - if !p.Modules["mesh-delivery"].Gate.Waiting() { - t.Fatal("a gate whose busy wait has not ended does not say it waits") - } - w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil) - busy := phase(w, PhaseBusy, 0) - if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") { - t.Fatalf("an open busy wait is not said: %+v", busy) - } - if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" { - t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases) - } -} - -// A container a report lists waiting is its module's by its id; its output alone is not a new account. -func TestABusyContainerIsItsModules(t *testing.T) { - list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"}, - {ID: "web.server", State: "replaced-when-idle"}} - if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" { - t.Fatalf("build-agent's container is not found waiting: %+v", b) - } - if _, ok := WaitingFor(list, "web"); ok { - t.Error("a container replaced reads as waiting") - } - if _, ok := WaitingFor(list, "build"); ok { - t.Error("another module's container is found by a prefix of its name") - } - again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]} - if !sameBusy(list, again) { - t.Error("a wait whose output changed is a new account") - } - if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() { - t.Error("a wait that ended is not a new account") - } -} - -// A machine's report of a busy container is kept with its last account and read back beside it, and its -// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store. - -// A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept -// out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335). -func TestARestMachineWaitingForABusyContainerIsNeverSilent(t *testing.T) { - c := walk582(t) - shanks := c.walk.Modules["build-agent"].Rest["shanks"] - ref := shanks.Digest + "@" + shanks.At.Format("2006-01-02T15:04:05.000000") - answered := shanks.At.Add(15 * time.Second) - // shanks answered, and said a container waits: its apply row is the answer, marked busy. - c.rows["shanks"][0].Detail = ApplyBusy - at := c.judged.Add(20 * time.Minute) // past ApplySilentAfter from its send - w := c.walk.WalkPhases(at, c.lookup) - var got RestReport - for _, r := range w.Rest { - if r.Node == "shanks" { - got = r - } - } - if got.State != RestBusy || slicesContains(w.Silent, "shanks") { - t.Fatalf("a machine waiting for a busy container reads %q (silent: %v)", got.State, w.Silent) - } - if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waits for a busy container on shanks") { - t.Fatalf("the walk ended, or does not say it waits: %+v", phase(w, PhaseApply, -1)) - } - - // 17 minutes on, shanks says it replaced the container: its wait is its own duration, and its report is then. - c.rows["shanks"] = append(c.rows["shanks"], Duration{Kind: DurationBusy, Subject: "shanks", Node: "shanks", Ref: ref, - Started: answered, Took: 17 * time.Minute}) - w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - for _, r := range w.Rest { - if r.Node == "shanks" { - got = r - } - } - if got.State != RestApplied || got.Busy != "17m0s" || !got.Reported.Equal(answered.Add(17*time.Minute)) { - t.Fatalf("a machine replaced after its wait reads %+v", got) - } - if slicesContains(w.Silent, "shanks") || !w.End.Equal(answered.Add(17*time.Minute)) { - t.Fatalf("the walk read shanks as silent, or ended elsewhere: %v %v", w.Silent, w.End) - } - // And the apply time itself is the answer's, never the wait's. - if r, _ := FirstReportOf(c.rows["shanks"], shanks, c.walkSent); !r.Answered.Equal(answered) || r.Busy != 17*time.Minute { - t.Fatalf("the busy wait is not kept apart from the apply: %+v", r) - } -} - -func slicesContains(list []string, s string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} diff --git a/internal/inventory/c5_still_reads_test.go b/internal/inventory/c5_still_reads_test.go new file mode 100644 index 00000000..3583a088 --- /dev/null +++ b/internal/inventory/c5_still_reads_test.go @@ -0,0 +1,32 @@ +//go:build integration + +package inventory + +import "testing" + +// **The store after release 6's migrations is still one today's controller (C5) can read** (issue 1945, review round 1): the +// revert path puts C5 back, either by the witness or by the operator's want, and it reads the columns of the node row that +// recorded a send (sent, sent_at, sent_bus_users, sent_builds, sent_epoch, sent_summary, sent_generation, sent_provisioning) +// and the tables of the plans' machinery (build_gate, declaration_send, assignment_generation, duration). Release 6 reads +// none of them any more and drops none: they go in a later milestone, once C5 has left the controller's five newest +// fingerprints. Each query below is one C5 ran. +func TestTheStoreAfterReleaseSixIsStillOneTheControllerOfBeforeCanRead(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + for _, q := range []string{ + `select sent, sequence, sent_at, sent_provisioning from node`, + `select sent_bus_users, sent_builds, sent_epoch, sent_summary, sent_generation from node`, + `select name, sent, sent_at from node order by name`, + `select build from build_gate where verdict = 'failed'`, + `select count(*) from declaration_send`, + `select generation from assignment_generation`, + `select kind, subject, ref, took_ms from duration`, + } { + if _, err := inv.store.Pool().Exec(ctx, q); err != nil { + t.Errorf("a query of the controller of before no longer runs on the store: %s: %v", q, err) + } + } +} diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index 0d297639..6f5d8af0 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -25,10 +25,6 @@ var ErrNoChain = errors.New("this kind of record lives on the bus, and this proc type chainState struct { mu sync.Mutex open ChainOpener - // view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that - // is not saved yet. - view *rolloutView - pending map[string]pendingMerge // requireLayout is set for a chain on the bus: its builds are read only once the controller's start has laid them out // under their modules (issue 1121), and laid is that seen. requireLayout, laid bool @@ -75,10 +71,6 @@ func (i *Inventory) UseChain(c *chain.Chain) { i.state.mu.Lock() defer i.state.mu.Unlock() i.chain = c - if i.state.view != nil { - i.state.view.stop() - i.state.view = nil - } } // UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go index ff7b8a02..1d56aae6 100644 --- a/internal/inventory/chain_assignments.go +++ b/internal/inventory/chain_assignments.go @@ -9,8 +9,6 @@ import ( "strings" "time" - "github.com/jackc/pgx/v5" - "github.com/novox/mesh-controller/internal/chain" ) @@ -124,13 +122,6 @@ func (i *Inventory) isAssigned(ctx context.Context, node, module string) (bool, return false, nil } -// raiseGeneration raises the assignment generation, as the store's triggers do for a change of its table (migration -// 0094): in the chain nothing in the store changes with an assignment, so the writer says it. -func (i *Inventory) raiseGeneration(ctx context.Context) error { - _, err := i.store.Pool().Exec(ctx, `update assignment_generation set generation = generation + 1`) - return err -} - // chainAssign puts a module on a machine, and says whether that is new. func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, module string) (bool, error) { var known bool @@ -154,7 +145,7 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, switch { case found && cur.Value.Stands(): // A module only pulled in by another becomes an assignment, keeping what it holds. - cur.Value.Assigned, cur.Value.PulledIn = now, false + cur.Value.Assigned = now _, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision) case found: // One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one @@ -170,9 +161,6 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, if err != nil { return false, fmt.Errorf("%s is not assigned to %s on the bus: %w", module, node.Name, err) } - if err := i.raiseGeneration(ctx); err != nil { - return true, err - } return true, nil } return false, fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name) @@ -195,41 +183,11 @@ func (i *Inventory) chainUnassign(ctx context.Context, c *chain.Chain, node Node } else if err != nil { return fmt.Errorf("%s is not taken off %s on the bus: %w", module, node.Name, err) } - return i.raiseGeneration(ctx) + return nil } return fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name) } -// chainAwaitingSince is AwaitingSince over the bucket: the oldest assignment on the machine made after it was last sent. -func (i *Inventory) chainAwaitingSince(ctx context.Context, name string) (time.Time, error) { - var sentAt *time.Time - var created time.Time - err := i.store.Pool().QueryRow(ctx, `select sent_at, created from node where name = $1`, name).Scan(&sentAt, &created) - if errors.Is(err, pgx.ErrNoRows) { - return time.Time{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name) - } - if err != nil { - return time.Time{}, err - } - rows, err := i.assignments(ctx) - if err != nil { - return time.Time{}, err - } - var oldest time.Time - for _, a := range rows { - if a.Node == name && (sentAt == nil || a.Assigned.After(*sentAt)) && (oldest.IsZero() || a.Assigned.Before(oldest)) { - oldest = a.Assigned - } - } - switch { - case !oldest.IsZero(): - return oldest, nil - case sentAt != nil: - return *sentAt, nil - } - return created, nil -} - // KeepAssignmentPlans writes the provisioner's plan of each assignment into its value on the bus, and a value of its own for // each module only pulled in by another on a machine (issue 765), and ends the value of one that is no longer pulled in. The // plans are derived from what each machine was last sent, not part of the record of what is assigned, so they are read @@ -273,11 +231,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i case keepSkip: continue case keepCreate: - next.Assigned, next.PulledIn = time.Time{}, true + next.Assigned = time.Time{} _, err = c.Assignments.Create(ctx, key, next) case keepFresh: // A value that ended, pulled in again: a fresh one over it, the ended one staying in history. - next.Assigned, next.PulledIn = time.Time{}, true + next.Assigned = time.Time{} _, err = c.Assignments.Update(ctx, key, next, cur.Revision) default: cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning @@ -292,8 +250,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i written++ } for key, cur := range have { - if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" { - continue // one a send still carries is on the machine, and ends when a send does not carry it any more + if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() { + continue } // A value of a module no longer pulled in, which no assignment names: it ends, and stays. cur.Value.Ended = time.Now().UTC() diff --git a/internal/inventory/chain_builds.go b/internal/inventory/chain_builds.go index 03517eac..4794adc2 100644 --- a/internal/inventory/chain_builds.go +++ b/internal/inventory/chain_builds.go @@ -810,33 +810,11 @@ func fillManifests(ctx context.Context, c *chain.Chain, builds []Build) error { return nil } -// failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097). -func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) { - rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`) - if err != nil { - return nil, err - } - defer rows.Close() - out := map[string]bool{} - for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { - return nil, err - } - out[id] = true - } - return out, rows.Err() -} - func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) { all, err := chainBuildsOf(ctx, c, module) if err != nil { return nil, nil, err } - failed, err := i.failedAtAGate(ctx) - if err != nil { - return nil, nil, err - } ok := worked(all) byModule(ok) out := map[string][]sourceRow{} @@ -853,7 +831,7 @@ func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module order = append(order, b.Module) } out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint, - made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest, failedGate: failed[b.ID]}) + made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest}) } return out, order, nil } @@ -1104,8 +1082,7 @@ func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain) // chainHeldMade is, per module, what the build standing for its newest successful build made. Read from the pointers: the // module's `worked` (its newest success) and `held` (the first build of the run of one source that it stands on), one record -// of each. A build that failed its gate ends a run (issue 280), and the pointer cannot say where; then the module's builds -// are read. +// of each. func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[string][]Artifact, error) { worked, err := i.pointedBuilds(ctx, c, "*.worked") if err != nil { @@ -1115,29 +1092,12 @@ func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[stri if err != nil { return nil, err } - failed, err := i.failedAtAGate(ctx) - if err != nil { - return nil, err - } out := map[string][]Artifact{} for module, w := range worked { stand := w - if h, ok := held[module]; ok && !failed[w.ID] { + if h, ok := held[module]; ok { stand = h } - if failed[stand.ID] && stand.ID != w.ID { - rows, _, err := i.chainSourceRows(ctx, c, module) - if err != nil { - return nil, err - } - if builds := rows[module]; len(builds) > 0 { - var made []Artifact - if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err == nil { - out[module] = made - } - } - continue - } out[module] = fromChainBuild(stand).Made } return out, nil diff --git a/internal/inventory/chain_builds_test.go b/internal/inventory/chain_builds_test.go index b1ef3314..0d8c5ddb 100644 --- a/internal/inventory/chain_builds_test.go +++ b/internal/inventory/chain_builds_test.go @@ -123,7 +123,6 @@ func TestTheBusBeingAwayIsNotNowLikeTheStoreBeingAway(t *testing.T) { for name, err := range map[string]error{ "a value too large": chain.ErrTooLarge, "a key not taken": chain.ErrBadKey, - "a plan moved": ErrPlanMoved, "a key that exists": chain.ErrExists, } { if Unreachable(err) { diff --git a/internal/inventory/chain_first_read_lays_out_test.go b/internal/inventory/chain_first_read_lays_out_test.go index 7b80c80f..ef11f3ad 100644 --- a/internal/inventory/chain_first_read_lays_out_test.go +++ b/internal/inventory/chain_first_read_lays_out_test.go @@ -23,3 +23,13 @@ func TestTheFirstReadOfAChainNotLaidOutLaysItOut(t *testing.T) { t.Fatalf("the read did not lay the builds out: found %v, %v", found, err) } } + +// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the +// store. +func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) { + t.Helper() + c := chain.InMemory(nil) + inv := &Inventory{state: &chainState{}} + inv.UseChain(c) + return inv, c +} diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go index 37f8f3b7..9bd85ce7 100644 --- a/internal/inventory/chain_links.go +++ b/internal/inventory/chain_links.go @@ -2,15 +2,7 @@ package inventory import ( "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" "sort" - "time" - - "github.com/jackc/pgx/v5" "github.com/novox/mesh-controller/internal/chain" ) @@ -21,44 +13,6 @@ import ( // release_plan.go). One read of an assignment then names the commit, the build and the rollout that put it there, and one read // of the rollout lists its assignments. -type rolloutKey struct{} - -// RolloutModule is what a rollout sends of one module: the commit and the build of it. -type RolloutModule struct{ Commit, Build string } - -type rolloutOf struct { - id string - modules map[string]RolloutModule -} - -// WithRollout is a context for a send a rollout makes: what RecordSent writes in the assignments of the modules it carries -// names this rollout and the build it sent. A module the send carries beside these, that another walk is moving, is named -// by the build the mesh holds of its commit and by no rollout. -func WithRollout(ctx context.Context, id string, modules map[string]RolloutModule) context.Context { - return context.WithValue(ctx, rolloutKey{}, rolloutOf{id: id, modules: modules}) -} - -// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the -// commit. Empty when the assignments are not on the bus, and for a module not sent yet. -func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - all, err := c.Assignments.All(ctx) - if err != nil { - return nil, err - } - out := map[string]chain.Ref{} - for _, k := range all { - a := k.Value - if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") { - out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit} - } - } - return out, nil -} - // AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links. func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) { c, err := i.theChain(ctx) @@ -78,285 +32,3 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass sort.Slice(out, func(a, b int) bool { return out[a].Module < out[b].Module }) return out, nil } - -// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh -// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of -// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where -// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried -// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named, -// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs -// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for -// what a machine was sent is read from these keys and from nowhere else. -func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error { - c, err := i.theChain(ctx) - if err != nil { - return fmt.Errorf("what the machine runs cannot be written: %w", err) - } - var node string - if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) { - return nil - } else if err != nil { - return err - } - rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf) - all, err := c.Assignments.All(ctx) - if err != nil { - return fmt.Errorf("the assignments of %s cannot be read: %w", node, err) - } - var failed []error - for _, k := range sortedValues(all) { - a := k.Value - if a.Node != node || !a.Stands() { - continue - } - if builds[a.Module] != "" { - continue // carried, and its commit is written below - } - if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil { - failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err)) - } - } - modules := make([]string, 0, len(builds)) - for m := range builds { - modules = append(modules, m) - } - sort.Strings(modules) - for _, m := range modules { - if builds[m] == "" { - continue - } - if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil { - failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err)) - } - } - return errors.Join(failed...) -} - -func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] { - out := make([]chain.Kept[chain.Assignment], 0, len(all)) - for key, k := range all { - k.Key = key - out = append(out, k) - } - sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) - return out -} - -// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key -// says what the machine runs is unknown. -func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error { - for try := 0; try < casTries; try++ { - next := k.Value - next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{} - next.SentUnknown = time.Time{} - if hand { - next.SentUnknown = at.UTC() - } - if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout && - next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) && - next.SentUnknown.Equal(k.Value.SentUnknown) { - return nil - } - if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) { - cur, found, gerr := c.Assignments.Get(ctx, k.Key) - if gerr != nil { - return gerr - } - if !found || !cur.Value.Stands() { - return nil - } - k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision} - continue - } else { - return err - } - } - return fmt.Errorf("the assignment kept moving under the write") -} - -func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error { - key := assignmentKey(node, module) - var build, rolloutID string - if sent, ours := rollout.modules[module]; ours && sent.Commit == commit { - build, rolloutID = sent.Build, rollout.id - } - var published *chain.Published - resolved := false - for try := 0; try < casTries; try++ { - cur, found, err := c.Assignments.Get(ctx, key) - if err != nil { - return err - } - if found && cur.Value.State != "" { - return nil // a wait for a build: the module is not on the machine by it - } - standing := found && cur.Value.Stands() - same := standing && cur.Value.Commit == commit - if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") { - return nil // a send that carries what the machine runs already says nothing new - } - if !resolved { - // The build of the commit is looked up once, and only for a module that moved or names no build. - if build == "" { - if build, err = i.BuildOf(ctx, module, commit); err != nil { - return err - } - } - if build != "" { - if b, found, err := i.BuildByID(ctx, build); err != nil { - return err - } else if found { - published = publishedOf(b.Made) - } - } - resolved = true - } - next := cur.Value - switch { - case !standing: - // Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is - // written over, the ended one staying in history. - next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID, - Since: at.UTC()} - case same: - // Already running it: fill what is missing, blank nothing. - if next.Build == "" { - next.Build = build - } - if next.Rollout == "" { - next.Rollout = rolloutID - } - if next.Since.IsZero() { - next.Since = at.UTC() - } - default: - next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC() - next.Artifact, next.Package = "", nil - } - next.SentUnknown = time.Time{} - if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build { - next.Artifact, next.Package = published.Artifact, published.Package - } - if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout && - next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) && - next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) { - return nil - } - if !found { - if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) { - continue - } else { - return err - } - } - if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) { - continue - } else { - return err - } - } - return fmt.Errorf("the assignment kept moving under the write") -} - -// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of -// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto -// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the -// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys -// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended. -// It answers how many keys it wrote; an error is raised by the caller as a condition. -func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) { - c, err := i.theChain(ctx) - if err != nil { - return 0, err - } - rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`) - if err != nil { - return 0, err - } - type kept struct { - name string - sent map[string]string - } - var all []kept - for rows.Next() { - var name string - var raw []byte - if err := rows.Scan(&name, &raw); err != nil { - rows.Close() - return 0, err - } - sent := map[string]string{} - if err := json.Unmarshal(raw, &sent); err != nil { - rows.Close() - return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err) - } - all = append(all, kept{name, sent}) - } - rows.Close() - if err := rows.Err(); err != nil { - return 0, err - } - now := time.Now().UTC() - for _, k := range all { - if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil { - return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err) - } - if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil { - return 0, err - } - } - return len(all), nil -} - -// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that -// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is -// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are. -func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error { - for _, module := range slices.Sorted(maps.Keys(sent)) { - commit := sent[module] - if commit == "" { - continue - } - key := assignmentKey(node, module) - build, err := i.chainBuildOf(ctx, c, module, commit) - if err != nil { - return err - } - for try := 0; ; try++ { - if try == casTries { - return fmt.Errorf("the assignment %s kept moving under the write", key) - } - cur, found, err := c.Assignments.Get(ctx, key) - if err != nil { - return err - } - if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) { - break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown - } - next := cur.Value - if !found { - next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at} - } - next.Commit, next.Build = commit, build - if next.Since.IsZero() { - next.Since = at - } - if !found { - _, err = c.Assignments.Create(ctx, key, next) - if errors.Is(err, chain.ErrExists) { - continue - } - } else { - _, err = c.Assignments.Update(ctx, key, next, cur.Revision) - if errors.Is(err, chain.ErrMoved) { - continue - } - } - if err != nil { - return err - } - break - } - } - return nil -} diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go index 3d48ba19..b18acaa0 100644 --- a/internal/inventory/chain_pending_test.go +++ b/internal/inventory/chain_pending_test.go @@ -77,9 +77,6 @@ func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) { if running, _ := inv.Running(ctx, "web"); len(running) != 0 { t.Fatalf("running %v", running) } - if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 { - t.Fatalf("runs %v", runs) - } entries, _ := inv.Catalogued(ctx) for _, e := range entries { if e.Manifest.Module == "web" && len(e.On) != 0 { diff --git a/internal/inventory/chain_rollouts.go b/internal/inventory/chain_rollouts.go deleted file mode 100644 index 9bcfa996..00000000 --- a/internal/inventory/chain_rollouts.go +++ /dev/null @@ -1,986 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan -// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk -// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside. -// -// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value, -// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no -// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the -// merge operations alone; a save keeps what the record holds. - -// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the -// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by -// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one. -type rolloutView struct { - chain *chain.Chain - mu sync.RWMutex - entries map[string]viewEntry - ready chan struct{} - once sync.Once - err error - dead atomic.Bool - cancel context.CancelFunc -} - -// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out. -type viewEntry struct { - raw []byte - rev uint64 - sum planSummary -} - -// planSummary is the part of a plan the surveys filter and sort by. -type planSummary struct { - ID string `json:"id"` - State string `json:"state"` - Created time.Time `json:"created"` - Repository string `json:"repository"` - Branch string `json:"branch"` - Merged time.Time `json:"merged"` - Release json.RawMessage `json:"release"` - Held []chain.HeldMerge `json:"held"` -} - -func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling } - -func (s planSummary) isRelease() bool { - r := strings.TrimSpace(string(s.Release)) - return r != "" && r != "null" -} - -func newRolloutView(c *chain.Chain) *rolloutView { - return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})} -} - -func (v *rolloutView) start() { - ctx, cancel := context.WithCancel(context.Background()) - v.cancel = cancel - go func() { - w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx) - if err != nil { - v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err)) - return - } - for { - select { - case <-ctx.Done(): - v.fail(ctx.Err()) - return - case e, open := <-w: - if !open { - v.fail(errors.New("the rollouts bucket stopped answering")) - return - } - if e.Caught { - v.once.Do(func() { close(v.ready) }) - continue - } - v.apply(e.Key, e.Value, e.Revision) - } - } - }() -} - -// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read. -func (v *rolloutView) fail(err error) { - v.mu.Lock() - if v.err == nil { - v.err = err - } - v.mu.Unlock() - v.dead.Store(true) - v.once.Do(func() { close(v.ready) }) -} - -func (v *rolloutView) stop() { - if v.cancel != nil { - v.cancel() - } -} - -// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one. -func (v *rolloutView) apply(key string, raw []byte, rev uint64) { - var sum planSummary - if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" { - sum = planSummary{ID: key} - } - v.mu.Lock() - defer v.mu.Unlock() - if old, has := v.entries[key]; has && old.rev >= rev { - return - } - v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum} -} - -func (v *rolloutView) list() []viewEntry { - v.mu.RLock() - defer v.mu.RUnlock() - out := make([]viewEntry, 0, len(v.entries)) - for _, e := range v.entries { - out = append(out, e) - } - return out -} - -// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole. -func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) { - i.state.mu.Lock() - v := i.state.view - if v == nil || v.chain != c || v.dead.Load() { - if v != nil { - v.stop() - } - v = newRolloutView(c) - i.state.view = v - v.start() - } - i.state.mu.Unlock() - select { - case <-v.ready: - case <-ctx.Done(): - return nil, ctx.Err() - } - v.mu.RLock() - defer v.mu.RUnlock() - if v.err != nil && len(v.entries) == 0 { - return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err) - } - return v, nil -} - -// stopViews ends the view's watch: the inventory is closing. -func (i *Inventory) stopViews() { - i.state.mu.Lock() - defer i.state.mu.Unlock() - if i.state.view != nil { - i.state.view.stop() - i.state.view = nil - } -} - -func normalizePlan(p *Plan) { - if p.Modules == nil { - p.Modules = map[string]*PlanModule{} - } - if p.TierEntered.IsZero() { - p.TierEntered = p.Created - } - if !p.Merged.IsZero() { - p.Merged = p.Merged.UTC() - } -} - -func planOf(e viewEntry) (Plan, error) { - var p Plan - if err := json.Unmarshal(e.raw, &p); err != nil { - return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err) - } - normalizePlan(&p) - return p, nil -} - -func sortCreated(entries []viewEntry) { - sort.SliceStable(entries, func(a, b int) bool { - if !entries[a].sum.Created.Equal(entries[b].sum.Created) { - return entries[a].sum.Created.Before(entries[b].sum.Created) - } - return entries[a].sum.ID < entries[b].sum.ID - }) -} - -func sortCreatedNewest(entries []viewEntry) { - sort.SliceStable(entries, func(a, b int) bool { - if !entries[a].sum.Created.Equal(entries[b].sum.Created) { - return entries[a].sum.Created.After(entries[b].sum.Created) - } - return entries[a].sum.ID > entries[b].sum.ID - }) -} - -// chainPlans is the plans that pass keep, in an order, read from the view. -func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) { - v, err := i.rolloutView(ctx, c) - if err != nil { - return nil, err - } - var chosen []viewEntry - for _, e := range v.list() { - if e.sum.ID != WaitingRecord && keep(e) { - chosen = append(chosen, e) - } - } - order(chosen) - out := make([]Plan, 0, len(chosen)) - for _, e := range chosen { - p, err := planOf(e) - if err != nil { - return nil, err - } - out = append(out, p) - } - return out, nil -} - -func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) { - v, err := i.rolloutView(ctx, c) - if err != nil { - return Plan{}, false, err - } - var chosen []viewEntry - for _, e := range v.list() { - if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() { - chosen = append(chosen, e) - } - } - sort.SliceStable(chosen, func(a, b int) bool { - if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) { - return chosen[a].sum.Merged.After(chosen[b].sum.Merged) - } - return chosen[a].sum.Created.After(chosen[b].sum.Created) - }) - if len(chosen) == 0 { - return Plan{}, false, nil - } - p, err := planOf(chosen[0]) - return p, err == nil, err -} - -// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view -// learns of it. -func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) { - if !chain.ValidKey(id) { - // No plan has an id the bus takes no spelling of; it is a plain id no plan has. - return Plan{}, fmt.Errorf("no plan %s", id) - } - e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id) - if err != nil { - return Plan{}, err - } - if !found { - return Plan{}, fmt.Errorf("no plan %s", id) - } - if v, err := i.rolloutView(ctx, c); err == nil { - v.apply(id, e.Value, e.Revision) - } - return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}}) -} - -// ---- the merges - -// mergeHolder is a record's copy of a merge. -type mergeHolder struct { - plan string - m chain.HeldMerge - rev uint64 -} - -func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit } - -// mergeHolders is every record's copy of each merge, by the merge's key. -func mergeHolders(entries []viewEntry) map[string][]mergeHolder { - out := map[string][]mergeHolder{} - for _, e := range entries { - for _, m := range e.sum.Held { - k := mergeKey(m.Repository, m.Commit) - out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev}) - } - } - return out -} - -// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two -// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers -// before one no plan does, then the record written last. -func winner(copies []mergeHolder) mergeHolder { - best := copies[0] - for _, c := range copies[1:] { - switch { - case c.m.Placed.After(best.m.Placed): - best = c - case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting: - best = c - case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev: - best = c - } - } - return best -} - -func winners(entries []viewEntry) map[string]mergeHolder { - out := map[string]mergeHolder{} - for k, copies := range mergeHolders(entries) { - out[k] = winner(copies) - } - return out -} - -func batchedOf(h mergeHolder) BatchedMerge { - b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit, - Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone} - if !h.m.Merged.IsZero() { - b.Merged = h.m.Merged.UTC() - } - if h.m.Waiting { - b.Plan = "" - } - return b -} - -func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge { - h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(), - Event: m.Event, Alone: m.Alone, Placed: now.UTC()} - if !m.Merged.IsZero() { - h.Merged = m.Merged.UTC() - } - return h -} - -func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) { - v, err := i.rolloutView(ctx, c) - if err != nil { - return nil, err - } - return winners(v.list()), nil -} - -func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return BatchedMerge{}, false, err - } - h, ok := w[mergeKey(repository, commit)] - if !ok { - return BatchedMerge{}, false, nil - } - return batchedOf(h), true, nil -} - -// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard. -func sortMerges(ms []BatchedMerge, newest bool) { - sort.SliceStable(ms, func(a, b int) bool { - x, y := ms[a], ms[b] - switch { - case x.Merged.IsZero() != y.Merged.IsZero(): - return !x.Merged.IsZero() - case !x.Merged.Equal(y.Merged): - if newest { - return x.Merged.After(y.Merged) - } - return x.Merged.Before(y.Merged) - } - if newest { - return x.Heard.After(y.Heard) - } - return x.Heard.Before(y.Heard) - }) -} - -func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if h.plan == plan && !h.m.Waiting { - out = append(out, batchedOf(h)) - } - } - sortMerges(out, false) - return out, nil -} - -func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) { - out = append(out, batchedOf(h)) - } - } - sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) }) - return out, nil -} - -func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return false, time.Time{}, err - } - var first time.Time - for _, h := range w { - if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) { - first = h.m.Heard - } - } - if first.IsZero() { - return false, time.Time{}, nil - } - return true, first.UTC(), nil -} - -// chainEarliestKept is when the controller heard the first merge it holds a record of, on any repository and branch. -func (i *Inventory) chainEarliestKept(ctx context.Context, c *chain.Chain) (bool, time.Time, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return false, time.Time{}, err - } - var first time.Time - for _, h := range w { - if first.IsZero() || h.m.Heard.Before(first) { - first = h.m.Heard - } - } - if first.IsZero() { - return false, time.Time{}, nil - } - return true, first.UTC(), nil -} - -func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if h.m.Waiting && h.m.Alone { - out = append(out, batchedOf(h)) - } - } - sortMerges(out, true) - return out, nil -} - -// chainWaitingMerges is every merge no plan answers that waits and is not walked alone, the newest first. -func (i *Inventory) chainWaitingMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if h.m.Waiting && !h.m.Alone { - out = append(out, batchedOf(h)) - } - } - sortMerges(out, true) - return out, nil -} - -// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it. -type pendingMerge struct { - plan string - m chain.HeldMerge -} - -// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has -// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is -// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3). -func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) { - waiting := m.Plan == "" - if waiting { - // A merge no plan answers is held, flagged, by the record kept for those. - m.Plan = WaitingRecord - if err := i.ensureWaitingRecord(ctx, c); err != nil { - return false, err - } - } - v, err := i.rolloutView(ctx, c) - if err != nil { - return false, err - } - key := mergeKey(m.Repository, m.Commit) - if _, known := winners(v.list())[key]; known { - return false, nil - } - held := heldOf(m, time.Now()) - held.Waiting = waiting - found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { - for _, x := range h { - if mergeKey(x.Repository, x.Commit) == key { - return h, false - } - } - return append(h, held), true - }) - if err != nil { - return false, err - } - if !found { - i.state.mu.Lock() - if i.state.pending == nil { - i.state.pending = map[string]pendingMerge{} - } - i.state.pending[key] = pendingMerge{plan: m.Plan, m: held} - i.state.mu.Unlock() - } - return true, nil -} - -// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store -// kept as a row with a null plan. It is no plan: the surveys of plans leave it out. -const WaitingRecord = "waiting-merges" - -// ensureWaitingRecord makes the record of the merges no plan answers, once. -func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error { - b := chain.RolloutsOf[Plan](c) - if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found { - return err - } - body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1, - Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"}) - if err != nil { - return err - } - rev, err := b.CreateRaw(ctx, WaitingRecord, body) - if errors.Is(err, chain.ErrExists) { - return nil - } - if err != nil { - return err - } - if v, verr := i.rolloutView(ctx, c); verr == nil { - v.apply(WaitingRecord, body, rev) - } - return nil -} - -// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it. -var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved") - -// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again -// when it moved. It answers false when the plan has no record. -func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) { - b := chain.RolloutsOf[Plan](c) - for try := 0; try < casTries; try++ { - e, found, err := b.GetRaw(ctx, id) - if err != nil || !found { - return false, err - } - var record map[string]json.RawMessage - if err := json.Unmarshal(e.Value, &record); err != nil { - return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err) - } - var held []chain.HeldMerge - if raw, has := record["held"]; has { - if err := json.Unmarshal(raw, &held); err != nil { - return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err) - } - } - next, changed := edit(held) - if !changed { - return true, nil - } - if len(next) == 0 { - delete(record, "held") - } else { - raw, err := json.Marshal(next) - if err != nil { - return true, err - } - record["held"] = raw - } - body, err := json.Marshal(record) - if err != nil { - return true, err - } - rev, err := b.UpdateRaw(ctx, id, body, e.Revision) - if errors.Is(err, chain.ErrMoved) { - continue - } - if err != nil { - return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err) - } - if v, verr := i.rolloutView(ctx, c); verr == nil { - v.apply(id, body, rev) - } - return true, nil - } - return true, fmt.Errorf("the merges of plan %s kept moving under the write", id) -} - -// recordEdit is what the merges of one record gain, lose and have flagged in one act. -type recordEdit struct { - add []chain.HeldMerge - drop map[string]bool - flag map[string]chain.HeldMerge -} - -func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge { - out := make([]chain.HeldMerge, 0, len(held)+len(e.add)) - for _, h := range held { - k := mergeKey(h.Repository, h.Commit) - if e.drop[k] { - continue - } - if f, has := e.flag[k]; has { - h.Waiting, h.Alone = f.Waiting, f.Alone - // A merge put back to waiting may carry a new announcement (HoldMerge); every other flag copies its own. - if len(f.Event) > 0 { - h.Event = f.Event - } - } - out = append(out, h) - } - for _, a := range e.add { - k := mergeKey(a.Repository, a.Commit) - dup := false - for _, h := range out { - if mergeKey(h.Repository, h.Commit) == k { - dup = true - break - } - } - if !dup { - out = append(out, a) - } - } - return out -} - -func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 } - -// ---- the save - -// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save -// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one. -func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) { - if old == nil { - return now, nil // a new plan enters its first tier now - } - if old.State == PlanAssembling || old.State == PlanQueued { - return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276) - } - since := old.TierEntered - if since.IsZero() { - since = old.Created - } - wasOpen := old.State == PlanBuilding || old.State == PlanRolling - if !wasOpen && p.Open() { - // **A walk set open again enters its tier anew** (issue 1886): a resume or a retry at the tier it ended at. Kept, - // the entry of before its stop measured the tier from hours before the resume — R4e read "waiting … for 5h48m53s" - // seven minutes after it was resumed — and the watchdog of its progress (S3) found it late at once. - return now, nil - } - if !wasOpen || (old.Tier == p.Tier && p.Open()) { - return since, nil // still in the tier, or already ended - } - var modules []string - if old.Tier >= 0 && old.Tier < len(p.Tiers) { - modules = p.Tiers[old.Tier] - } - detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State) - if p.Open() { - detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier) - } - return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier), - Started: since, Took: now.Sub(since), Detail: detail} -} - -// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by -// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is -// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a -// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092). -func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) { - epoch, err := i.actingEpoch(ctx) - if err != nil { - if len(plans) > 0 { - return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err) - } - return err - } - v, err := i.rolloutView(ctx, c) - if err != nil { - return err - } - b := chain.RolloutsOf[Plan](c) - marked := time.Now() - undo := make([]func(), 0, len(plans)) - defer func() { - if err != nil { - for _, u := range undo { - u() - } - } - }() - for _, p := range plans { - undo = append(undo, p.markWalk(marked)) - } - - // Every plan still at the revision it was read at, before anything is written. - cur := make(map[string]chain.Kept[Plan], len(plans)) - for _, p := range plans { - k, found, err := b.Get(ctx, p.ID) - if err != nil { - return err - } - if found { - if k.Value.Revision != p.Revision { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - cur[p.ID] = k - } - } - - // What each record gains, loses and has flagged. - now := time.Now().UTC() - edits := map[string]*recordEdit{} - editOf := func(id string) *recordEdit { - if edits[id] == nil { - edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}} - } - return edits[id] - } - copies := mergeHolders(v.list()) - for _, m := range moves { - key := mergeKey(m.Repository, m.Commit) - held, has := copies[key] - if !has { - // A merge carried for a plan not saved yet goes where it is moved. - i.state.mu.Lock() - if pm, pending := i.state.pending[key]; pending && m.Plan != "" { - pm.plan, pm.m.Alone = m.Plan, m.Alone - i.state.pending[key] = pm - } - i.state.mu.Unlock() - continue - } - w := winner(held) - if m.Plan == "" { - f := w.m - f.Waiting, f.Alone = true, m.Alone - if m.Event != nil { - f.Event = m.Event - } - editOf(w.plan).flag[key] = f - continue - } - if w.plan == m.Plan { - f := w.m - f.Waiting, f.Alone = false, m.Alone - editOf(w.plan).flag[key] = f - continue - } - moved := w.m - moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now - editOf(m.Plan).add = append(editOf(m.Plan).add, moved) - for _, h := range held { - editOf(h.plan).drop[key] = true - } - } - - // The records not among the plans: receivers now, givers after. - for id, e := range edits { - if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 { - continue - } - e := e - found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{add: e.add}).apply(h), true - }) - if err != nil { - return err - } - if !found { - // Nothing is taken from the record that holds a merge to give it to a plan that has none. - return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add)) - } - } - - // The plans, those that gain a merge first. - order := append([]*Plan(nil), plans...) - sort.SliceStable(order, func(a, b int) bool { - x, y := edits[order[a].ID], edits[order[b].ID] - return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0) - }) - type written struct { - revision int64 - entered time.Time - held []chain.HeldMerge - duration *Duration - } - done := map[string]written{} - for _, p := range order { - var w written - for try := 0; ; try++ { - if try == casTries { - return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID) - } - k, found := cur[p.ID] - if try > 0 { - fresh, still, gerr := b.Get(ctx, p.ID) - if gerr != nil { - return gerr - } - if still && fresh.Value.Revision != p.Revision { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - k, found = fresh, still - cur[p.ID] = k - } - record := *p - var old *Plan - var held []chain.HeldMerge - if found { - o := k.Value - old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...) - record.Revision = k.Value.Revision + 1 - } else { - record.Revision = 1 - } - if e := edits[p.ID]; e != nil { - held = e.apply(held) - } - i.state.mu.Lock() - for key, pm := range i.state.pending { - if pm.plan == p.ID { - held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held) - _ = key - } - } - i.state.mu.Unlock() - record.Held = held - entered, duration := tierLeftOf(old, *p, time.Now()) - record.TierEntered, record.Updated = entered, time.Now().UTC() - if epoch != nil { - record.Epoch = uint64(*epoch) - } else { - record.Epoch = 0 - } - body, merr := json.Marshal(record) - if merr != nil { - return merr - } - var rev uint64 - var werr error - if found { - rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision) - } else { - rev, werr = createRaw(ctx, c, p.ID, body) - } - if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) { - if !found { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - continue - } - if werr != nil { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr) - } - v.apply(p.ID, body, rev) - w = written{revision: record.Revision, entered: entered, held: held, duration: duration} - break - } - done[p.ID] = w - } - - // The records that gave a merge up, now it is held by the one it went to. - for id, e := range edits { - if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) { - continue - } - e := e - if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true - }); err != nil { - return err - } - } - - i.state.mu.Lock() - for key, pm := range i.state.pending { - if _, saved := done[pm.plan]; saved { - delete(i.state.pending, key) - } - } - i.state.mu.Unlock() - for _, p := range plans { - w := done[p.ID] - p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held - if epoch != nil { - p.Epoch = uint64(*epoch) - } else { - p.Epoch = 0 - } - if w.duration != nil { - // After the write: the key of the duration is the plan and its tier, so a measurement made twice is one. - record := i.RecordDuration - if i.recorded != nil { - record = i.recorded - } - if derr := record(ctx, *w.duration); derr != nil { - fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr) - } - } - if PlanSaved != nil { - PlanSaved(i, *p) - } - } - return nil -} - -func planNamed(plans []*Plan, id string) bool { - for _, p := range plans { - if p.ID == id { - return true - } - } - return false -} - -// createRaw writes a plan's first record. -func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) { - return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body) -} - -// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired -// at the next cut (ADR 0408, issue 1092). It says how many it repaired. -func (i *Inventory) RepairMerges(ctx context.Context) (int, error) { - c, err := i.theChain(ctx) - if err != nil { - return 0, err - } - v, err := i.rolloutView(ctx, c) - if err != nil { - return 0, err - } - repaired := 0 - for key, copies := range mergeHolders(v.list()) { - if len(copies) < 2 { - continue - } - w := winner(copies) - for _, h := range copies { - if h.plan == w.plan { - continue - } - key := key - if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true - }); err != nil { - return repaired, err - } - repaired++ - } - } - return repaired, nil -} diff --git a/internal/inventory/chain_rollouts_test.go b/internal/inventory/chain_rollouts_test.go deleted file mode 100644 index d7e8b7f3..00000000 --- a/internal/inventory/chain_rollouts_test.go +++ /dev/null @@ -1,374 +0,0 @@ -package inventory - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the -// store but for the durations a save measures, taken here by nothing. -func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) { - t.Helper() - c := chain.InMemory(nil) - inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} - inv.UseChain(c) - t.Cleanup(inv.stopViews) - return inv, c -} - -var plansAt = time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - -func aBatch(id string, at time.Time) *Plan { - return &Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c-" + id, Created: at, State: PlanAssembling, - Tiers: [][]string{}, Modules: map[string]*PlanModule{}} -} - -func aMerge(commit, plan string) BatchedMerge { - return BatchedMerge{Repository: "Novox/Mesh-Catalog", Branch: "main", Commit: commit, Merged: plansAt, Heard: plansAt.Add(time.Second), - Event: []byte(`{"commit":"` + commit + `"}`), Plan: plan} -} - -// **A plan is kept under its id; a save of a copy another save has moved since is refused and writes nothing** (ADR 0408 -// §2, issue 1092): the plan's own Revision counts its saves, 1 for the first. -func TestAPlanIsKeptUnderItsIdAndASaveOfAStaleCopyIsRefused(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - p := aBatch("plan-1", plansAt) - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - if p.Revision != 1 { - t.Fatalf("a new plan is at revision %d", p.Revision) - } - stale := *p - p.Note = "moved on" - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil || p.Revision != 2 { - t.Fatalf("the second save: revision %d, %v", p.Revision, err) - } - stale.Note = "laid over" - if err := inv.chainSavePlans(ctx, c, []*Plan{&stale}, nil); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a stale copy was saved: %v", err) - } - got, err := inv.chainPlanByID(ctx, c, "plan-1") - if err != nil || got.Note != "moved on" || got.Revision != 2 { - t.Fatalf("the plan is %+v: %v", got, err) - } - // A plan new to the bucket that is already there is refused too. - again := aBatch("plan-1", plansAt) - if err := inv.chainSavePlans(ctx, c, []*Plan{again}, nil); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a new plan over a kept one: %v", err) - } - if _, err := inv.chainPlanByID(ctx, c, "plan-nobody"); err == nil || err.Error() != "no plan plan-nobody" { - t.Fatalf("a plan nobody kept: %v", err) - } -} - -// **A save keeps the merges the record holds, not the ones its copy read** (issue 1092): a merge that moved into the record -// after the copy was read is not laid back by the copy's save. -func TestASaveKeepsTheMergesTheRecordHoldsAndNotTheCopys(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - batch := aBatch("plan-1", plansAt) - if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil { - t.Fatal(err) - } - if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || !added { - t.Fatalf("added %v: %v", added, err) - } - // The copy was read before the merge came in; saving it neither fails nor loses the merge. - batch.Note = "window closed" - if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil { - t.Fatalf("a copy read before a merge came in: %v", err) - } - merges, err := inv.chainMergesOf(ctx, c, "plan-1") - if err != nil || len(merges) != 1 || merges[0].Commit != "m1" || merges[0].Repository != "novox/mesh-catalog" { - t.Fatalf("the merges of the plan: %+v %v", merges, err) - } - if string(merges[0].Event) != `{"commit":"m1"}` || merges[0].Heard.IsZero() || merges[0].Plan != "plan-1" { - t.Fatalf("the merge came back as %+v", merges[0]) - } - // A merge heard is not heard twice, whatever the spelling of its repository. - if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || added { - t.Fatalf("added again %v: %v", added, err) - } -} - -// **A merge added to a plan that is not saved is written by that plan's save, and by nothing else** (issue 1092): until -// then it is in no record — so the event that brought it is handed over again if the save fails. -func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-new")); err != nil || !added { - t.Fatalf("added %v: %v", added, err) - } - if _, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); known { - t.Fatal("a merge for a plan not saved is in a record") - } - // The save is refused: nothing is written, the merge is still carried. - held := false - away := chain.New(chain.NewMemory(), chain.NewMemory(), chain.NewMemory(), func(context.Context) error { - if held { - return nil - } - return errors.New("the lease is not held") - }) - other := &Inventory{state: &chainState{}, recorded: inv.recorded} - other.UseChain(away) - t.Cleanup(other.stopViews) - other.state.pending = inv.state.pending - p := aBatch("plan-new", plansAt) - if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) { - t.Fatalf("the save answered %v", err) - } - if _, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); known { - t.Fatal("a refused save left the merge in a record") - } - held = true - if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - if m, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); !known || m.Plan != "plan-new" { - t.Fatalf("the save did not write the merge: %+v %v", m, known) - } -} - -// failingOn is a bucket that refuses to write one key: the second half of a move, interrupted. -type failingOn struct { - chain.Backend - key string - fail *atomic.Bool -} - -func (f failingOn) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { - if key == f.key && f.fail.Load() { - return 0, errors.New("the bus went away") - } - return f.Backend.Update(ctx, key, value, revision) -} - -// **A merge moved to a plan is written to the receiving record first, then taken from the giving one; one found in both is -// read from the one it entered last and repaired** (issue 1092, ADR 0299 §7). -func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *testing.T) { - c := chain.InMemory(nil) - var fail atomic.Bool - rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail} - c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} - inv.UseChain(c) - t.Cleanup(inv.stopViews) - ctx := context.Background() - a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute)) - for _, p := range []*Plan{a, b} { - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - } - if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil { - t.Fatal(err) - } - - // The move stops before the giving record is written: the merge is in both. - fail.Store(true) - err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b"}}) - if err == nil { - t.Fatal("the move was said to be made with the giving record unwritten") - } - fail.Store(false) - if m, _, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); m.Plan != "plan-b" { - t.Fatalf("a merge in two records is read from %q, not the one it moved to", m.Plan) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 { - t.Fatalf("the giving plan still answers %+v", got) - } - // The next cut repairs it. - v, _ := inv.rolloutView(ctx, c) - if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 { - t.Fatalf("the merge is held by %d records, want the two of a stopped move", n) - } - repaired := 0 - for k, copies := range mergeHolders(v.list()) { - if len(copies) < 2 { - continue - } - w := winner(copies) - for _, h := range copies { - if h.plan != w.plan { - key := k - if _, err := inv.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true - }); err != nil { - t.Fatal(err) - } - repaired++ - } - } - } - if repaired != 1 { - t.Fatalf("repaired %d", repaired) - } - waitFor(t, func() bool { - v, _ := inv.rolloutView(ctx, c) - return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1 - }) -} - -// waitFor polls a condition that a watch makes true, for a moment. -func waitFor(t *testing.T, ok func() bool) { - t.Helper() - for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { - if ok() { - return - } - } - t.Fatal("the condition did not come true") -} - -// **A merge no plan answers stays, flagged, in the record that held it last, and is walked alone** (issue 1092): not in -// the merges of the plan, listed among the merges waiting, and answered again by the plan it is moved to. -func TestAWaitingMergeStaysInTheRecordThatHeldItLast(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute)) - for _, p := range []*Plan{a, b} { - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - } - if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil { - t.Fatal(err) - } - if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "", Alone: true}}); err != nil { - t.Fatal(err) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 { - t.Fatalf("a waiting merge is answered by %+v", got) - } - alone, err := inv.chainAloneMerges(ctx, c) - if err != nil || len(alone) != 1 || alone[0].Commit != "m1" || alone[0].Plan != "" || !alone[0].Alone { - t.Fatalf("waiting: %+v %v", alone, err) - } - if m, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); !known || m.Plan != "" { - t.Fatalf("a waiting merge reads as %+v %v", m, known) - } - if later, _ := inv.chainLaterMergesOf(ctx, c, "novox/mesh-catalog", "main", plansAt.Add(-time.Hour)); len(later) != 0 { - t.Fatalf("a waiting merge is among those a plan answers: %+v", later) - } - if kept, first, _ := inv.chainKeptOnBranch(ctx, c, "novox/mesh-catalog", "main"); !kept || first.IsZero() { - t.Fatalf("a waiting merge is not kept on its branch: %v %v", kept, first) - } - // Walked alone: the plan it goes to answers it, and the record that held it lets it go. - if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b", Alone: true}}); err != nil { - t.Fatal(err) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 || !got[0].Alone { - t.Fatalf("plan-b answers %+v", got) - } - if left, _ := inv.chainAloneMerges(ctx, c); len(left) != 0 { - t.Fatalf("still waiting: %+v", left) - } - waitFor(t, func() bool { - v, _ := inv.rolloutView(ctx, c) - return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1 - }) - // A move to a plan with no record is refused and takes nothing from the one that holds the merge. - err = inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-nowhere"}}) - if !errors.Is(err, ErrMergeUnsaved) { - t.Fatalf("a move to a plan with no record: %v", err) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 { - t.Fatalf("the merge was taken from plan-b: %+v", got) - } -} - -// countingBackend counts the reads a bucket answers. -type countingBackend struct { - chain.Backend - gets, keys atomic.Int32 -} - -func (c *countingBackend) Get(ctx context.Context, key string) (chain.Entry, bool, error) { - c.gets.Add(1) - return c.Backend.Get(ctx, key) -} - -func (c *countingBackend) Keys(ctx context.Context) ([]string, error) { - c.keys.Add(1) - return c.Backend.Keys(ctx) -} - -// **The surveys are read from a view kept by one watch, never from the bucket key by key** (issue 1092): the plans and the -// merges are asked for many times and the bucket is read for none of them; a write by another process reaches the view -// through its watch. -func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) { - counting := &countingBackend{Backend: chain.NewMemory()} - c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} - inv.UseChain(c) - t.Cleanup(inv.stopViews) - ctx := context.Background() - for k, id := range []string{"plan-1", "plan-2", "plan-3"} { - p := aBatch(id, plansAt.Add(time.Duration(k)*time.Minute)) - if k > 0 { - p.State = PlanBuilding - } - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - } - before := counting.gets.Load() - for n := 0; n < 20; n++ { - open, err := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) - if err != nil || len(open) != 2 || open[0].ID != "plan-2" { - t.Fatalf("open plans %+v: %v", open, err) - } - if _, err := inv.chainAloneMerges(ctx, c); err != nil { - t.Fatal(err) - } - } - if counting.gets.Load() != before || counting.keys.Load() != 0 { - t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load()) - } - // Another process writes; the view follows. - other := &Inventory{state: &chainState{}, recorded: inv.recorded} - other.UseChain(c) - t.Cleanup(other.stopViews) - late := aBatch("plan-4", plansAt.Add(time.Hour)) - late.State = PlanRolling - if err := other.chainSavePlans(ctx, c, []*Plan{late}, nil); err != nil { - t.Fatal(err) - } - waitFor(t, func() bool { - open, _ := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) - return len(open) == 3 - }) -} - -// Where the record of merges begins is the earliest merge kept on any repository and branch (novox/hq ADR 0463 §2). -func TestTheRecordOfMergesBeginsAtTheEarliestMergeKeptOnAnyTrunk(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - if kept, _, err := inv.chainEarliestKept(ctx, c); err != nil || kept { - t.Fatalf("an inventory with no merge kept says it keeps one: %v %v", kept, err) - } - if err := inv.chainSavePlans(ctx, c, []*Plan{aBatch("plan-1", plansAt), aBatch("plan-2", plansAt)}, nil); err != nil { - t.Fatal(err) - } - later := aMerge("m-late", "plan-1") - later.Repository, later.Branch, later.Heard = "novox/mesh-sdk", "main", plansAt.Add(48*time.Hour) - earlier := aMerge("m-early", "plan-2") - earlier.Repository, earlier.Branch, earlier.Heard = "novox/photos", "nox-mesh", plansAt.Add(time.Hour) - for _, m := range []BatchedMerge{later, earlier} { - if added, err := inv.chainAddMerge(ctx, c, m); err != nil || !added { - t.Fatalf("%s: %v %v", m.Commit, added, err) - } - } - kept, first, err := inv.chainEarliestKept(ctx, c) - if err != nil || !kept || !first.Equal(plansAt.Add(time.Hour)) { - t.Fatalf("the record begins at %v (%v, %v), want %v", first, kept, err, plansAt.Add(time.Hour)) - } -} diff --git a/internal/inventory/chain_sent_copy_test.go b/internal/inventory/chain_sent_copy_test.go deleted file mode 100644 index 3920c150..00000000 --- a/internal/inventory/chain_sent_copy_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package inventory - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review -// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy -// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build -// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build. -func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := t.Context() - at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC) - if err := inv.chainRecordBuild(ctx, c, Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil { - t.Fatal(err) - } - put := func(module string, a chain.Assignment) { - a.Node, a.Module = "laptop", module - if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil { - t.Fatal(err) - } - } - put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet - put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one - put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since - put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build - put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned - sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand", - "waits": "c-waits", "ended": "c-ended"} - - for pass := 0; pass < 2; pass++ { // and the second pass changes nothing - if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil { - t.Fatal(err) - } - get := func(module string) chain.Assignment { - t.Helper() - k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module)) - if err != nil || !found { - t.Fatalf("%s: %v %v", module, found, err) - } - return k.Value - } - if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn { - t.Errorf("notes: %+v", a) - } - if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" { - t.Errorf("a key with a commit was overwritten: %+v", a) - } - if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() { - t.Errorf("a hand send's unknown was overwritten: %+v", a) - } - if a := get("waits"); a.Commit != "" || a.State != chain.StatePending { - t.Errorf("a wait was overwritten: %+v", a) - } - if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" { - t.Errorf("the pulled-in module has no standing key of its own: %+v", a) - } - if a := get("ended"); a.Commit != "" || a.Stands() { - t.Errorf("an ended key was revived or given a commit: %+v", a) - } - } -} - -// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the -// pulled-in module, and a second copy finds nothing to do. -func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - if _, err := inv.AddNode(ctx, "laptop"); err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`) - if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 { - t.Fatalf("the copy did %d nodes (%v)", n, err) - } - got, known, err := inv.SentBuilds(ctx, "laptop") - if err != nil || !known || got["mesh-wireguard"] != "c-wg" { - t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err) - } - if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { - t.Fatalf("the column was not emptied: %v", rows) - } - if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { - t.Fatalf("a second copy did %d nodes (%v)", n, err) - } -} - -// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared -// without a commit. -func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`) - if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil { - t.Fatal(err) - } - if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { - t.Fatalf("a send left the column: %v", rows) - } - if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { - t.Fatalf("the copy did %d nodes (%v)", n, err) - } -} diff --git a/internal/inventory/chain_sent_test.go b/internal/inventory/chain_sent_test.go deleted file mode 100644 index a62ea017..00000000 --- a/internal/inventory/chain_sent_test.go +++ /dev/null @@ -1,116 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "errors" - "reflect" - "testing" - - "github.com/novox/mesh-controller/internal/chain" -) - -// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned -// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state. -func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) { - inv, node := aNodeWithModules(t, "web", "cache") - ctx := t.Context() - record, _ := inv.NodeByName(ctx, node) - if _, err := inv.Assign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) { - t.Fatalf("sent %v %v %v", builds, known, err) - } - pulled, found, _ := inv.AssignmentValue(ctx, node, "cache") - if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" { - t.Fatalf("the module only carried has the key %+v", pulled) - } - if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) { - t.Fatalf("a module only pulled in is assigned: %v", on) - } - // Assigned later, it keeps what it was sent and is pulled in no longer. - if _, err := inv.Assign(ctx, node, "cache"); err != nil { - t.Fatal(err) - } - made, _, _ := inv.AssignmentValue(ctx, node, "cache") - if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" { - t.Fatalf("assigned over a pulled-in key: %+v", made) - } -} - -// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds** -// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so. -func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) { - inv, node := aNodeWithModules(t, "web", "cache") - ctx := t.Context() - record, _ := inv.NodeByName(ctx, node) - if _, err := inv.Assign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil { - t.Fatal(err) - } - if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) { - t.Fatalf("the second send did not replace the map: %v %v", builds, known) - } - if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" { - t.Fatalf("a module no longer carried keeps %+v", old) - } - // By hand: unknown, with a time and no commit anywhere. - if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil { - t.Fatal(err) - } - if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil { - t.Fatalf("a hand send reads as %v %v", builds, known) - } - web, _, _ := inv.AssignmentValue(ctx, node, "web") - if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() { - t.Fatalf("web after a hand send: %+v", web) - } - if sent, _ := inv.Outstanding(ctx, node); sent != "d3" { - t.Fatalf("the digest of the hand send: %q", sent) - } - // A send that names its builds makes it known again. - if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil { - t.Fatal(err) - } - if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) { - t.Fatalf("after a send that names its builds: %v %v", builds, known) - } - if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() { - t.Fatalf("web still says unknown: %+v", again) - } -} - -// failingToWrite is an assignments bucket that fails every write. -type failingToWrite struct{ chain.Backend } - -func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) { - return 0, errors.New("the bus went away") -} - -func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) { - return 0, errors.New("the bus went away") -} - -// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the -// keys, so a send recorded without them would read as one that never happened. -func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) { - inv, node := aNodeWithModules(t, "web") - ctx := t.Context() - record, _ := inv.NodeByName(ctx, node) - inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()}, - func(context.Context) error { return nil })) - err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"}) - if err == nil { - t.Fatal("a send whose links could not be written was recorded") - } -} diff --git a/internal/inventory/chain_store_test.go b/internal/inventory/chain_store_test.go index d05be0d2..a05465f3 100644 --- a/internal/inventory/chain_store_test.go +++ b/internal/inventory/chain_store_test.go @@ -4,6 +4,7 @@ package inventory import ( "context" + "encoding/json" "errors" "reflect" "slices" @@ -72,10 +73,6 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { if err != nil || len(announced) != 3 { t.Fatalf("announceable %d: %v", len(announced), err) } - prints, err := inv.Fingerprints(ctx) - if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { - t.Fatalf("fingerprints %v: %v", prints, err) - } } // A build request row as the store held it before the move. @@ -166,7 +163,7 @@ func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) // **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a // repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module -// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the +// may be forgotten, what is behind — read the bucket, and a seat held by the // assignment goes with it. func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { inv, node := aNodeWithModules(t, "postgres", "web") @@ -174,7 +171,6 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { t.Fatal(err) } - start := generationNow(t, inv) if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added { t.Fatalf("assigned %v: %v", added, err) } @@ -185,8 +181,24 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() { t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err) } - if generationNow(t, inv) <= start { - t.Fatal("an assignment left the generation where it was") + // **Placement only** (ADR 0539 §12): the value on the bus names the node, the module and since when, as mesh-config + // reads them (unknown fields ignored there), and no build, commit or rollout. + kv, err := inv.KeptAssignments(ctx) + if err != nil || len(kv["anchor.postgres"].Value) == 0 { + t.Fatalf("the kept assignments: %v %v", kv, err) + } + var read struct { + Node string `json:"node"` + Module string `json:"module"` + Assigned time.Time `json:"assigned"` + } + if err := json.Unmarshal(kv["anchor.postgres"].Value, &read); err != nil || read.Node != "anchor" || read.Module != "postgres" || read.Assigned.IsZero() { + t.Fatalf("a reader of placement reads %+v from %s: %v", read, kv["anchor.postgres"].Value, err) + } + for _, not := range []string{`"build"`, `"commit"`, `"rollout"`, `"artifact"`, `"package"`, `"since"`} { + if strings.Contains(string(kv["anchor.postgres"].Value), not) { + t.Errorf("an assignment names %s: %s", not, kv["anchor.postgres"].Value) + } } if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) { t.Fatalf("a module the mesh does not know: %v", err) @@ -224,13 +236,9 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { t.Fatalf("a seat handed to an assignment: %v", err) } - before := generationNow(t, inv) if err := inv.Unassign(ctx, node, "postgres"); err != nil { t.Fatal(err) } - if generationNow(t, inv) <= before { - t.Fatal("an unassignment left the generation where it was") - } // No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history. gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() { @@ -278,12 +286,12 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { } // **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the -// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// plan of a module the machine was sent is written into its assignment, another field of the value stays, // a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here // ends an assignment. // **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the -// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// plan of a module the machine was sent is written into its assignment, another field of the value stays, // a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here // ends an assignment. func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) { @@ -300,23 +308,14 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { - t.Fatal(err) - } - // A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this - // one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded. - cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - cur.Value.Build, cur.Value.Commit = "build-1", "c1" - if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { - t.Fatal(err) - } + sentPlans(t, inv, record.ID, raw) inv.SetProvisioningPlans(inv.SentProvisioningPlans) written, removed, err := inv.KeepAssignmentPlans(ctx) if err != nil || written != 2 || removed != 0 { t.Fatalf("kept: %d written, %d ended: %v", written, removed, err) } dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() { + if dunst.Value.Provisioning == nil || dunst.Value.Assigned.IsZero() { t.Fatalf("dunst's value is %+v", dunst.Value) } if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil { @@ -334,9 +333,7 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi } // Sent again without the pulled-in module: its value ends, the assignments stay. only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil { - t.Fatal(err) - } + sentPlans(t, inv, record.ID, only) if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 { t.Fatalf("ended %d: %v", removed, err) } @@ -351,3 +348,20 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi // **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value** // (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build // already is left as it is. + +// sentPlans gives a machine the plans it was last sent, as the column the mirror reads keeps them. +func sentPlans(t *testing.T, inv *Inventory, id string, plans []byte) { + t.Helper() + if _, err := inv.store.Pool().Exec(t.Context(), `update node set sent_provisioning = $2 where id = $1`, id, plans); err != nil { + t.Fatal(err) + } +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(raw) +} diff --git a/internal/inventory/collection.go b/internal/inventory/collection.go index 2edf865e..cc6f1eb0 100644 --- a/internal/inventory/collection.go +++ b/internal/inventory/collection.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "fmt" "slices" "sort" "strings" @@ -47,6 +48,9 @@ const ( KeptByRecentRecord = "recorded-lately" // KeptByOpenPlan: a plan that has not ended names the build that made it (novox/hq issue 1933). KeptByOpenPlan = "open-plan" + // KeptByWant: a want names the fingerprint of the build that made it, so a node may be told to run it (novox/hq + // ADR 0539 clause 1): with no plan, the want is what names a build a node runs or moves to. + KeptByWant = "wanted" ) // KeptLately is how long after a build is recorded its artifacts are kept whatever else the records say (novox/hq @@ -458,7 +462,8 @@ func keptLately(builds []Build, planned map[string]bool, now time.Time, within t return out } -// keptLatelyOrPlanned is keptLately of every build the records hold, against the plans that have not ended. +// keptLatelyOrPlanned is keptLately of every build the records hold, and the builds of every fingerprint a want names +// (no plan names a build any more, ADR 0539 §17; a want does, clause 1). func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]string, error) { c, err := i.theChain(ctx) if err != nil { @@ -468,29 +473,46 @@ func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]strin if err != nil { return nil, err } - plans, err := i.OpenPlans(ctx) - if err != nil { - return nil, err + out := keptLately(all, nil, time.Now().UTC(), i.lately()) + if i.wanted == nil { + return out, nil } - return keptLately(all, plannedBuilds(plans), time.Now().UTC(), i.lately()), nil -} - -// plannedBuilds is every build id these plans name: a module's build, and the builds a rollout carried. -func plannedBuilds(plans []Plan) map[string]bool { - out := map[string]bool{} - for _, p := range plans { - for _, m := range p.Modules { - if m != nil && m.Build != "" { - out[m.Build] = true + wanted, err := i.wanted(ctx) + if err != nil { + return nil, fmt.Errorf("the fingerprints the wants name: %w", err) + } + for reference, whys := range keptByWants(all, wanted) { + for _, why := range whys { + if !slices.Contains(out[reference], why) { + out[reference] = append(out[reference], why) } } - for _, id := range p.Builds { - out[id] = true + } + return out, nil +} + +// keptByWants is, of these builds, the artifacts of each that worked and whose module and fingerprint a want names +// (` `), each kept as wanted. +func keptByWants(builds []Build, wanted map[string]bool) map[string][]string { + out := map[string][]string{} + for _, b := range builds { + if !b.Worked() || b.SourceFingerprint == "" || !wanted[b.Module+" "+b.SourceFingerprint] { + continue + } + for _, a := range b.Made { + if reference := asRecorded(a.Reference); reference != "" && !slices.Contains(out[reference], KeptByWant) { + out[reference] = append(out[reference], KeptByWant) + } } } return out } +// SetWanted gives the keep set every fingerprint a want names, as ` ` (ADR 0539 clause 1). +func (i *Inventory) SetWanted(wanted func(ctx context.Context) (map[string]bool, error)) { + i.wanted = wanted +} + // digestIn is the `sha256:` a reference names, empty when it names none. func digestIn(reference string) string { for _, marker := range []string{"@sha256:", "/sha256:"} { diff --git a/internal/inventory/collection_wanted_test.go b/internal/inventory/collection_wanted_test.go new file mode 100644 index 00000000..05871bf0 --- /dev/null +++ b/internal/inventory/collection_wanted_test.go @@ -0,0 +1,37 @@ +package inventory + +import ( + "strings" + "testing" + "time" +) + +// 1945 (dev-r6-controller's finding, for ADR 0539 clause 1): with no plan left to name a build, the builds of every +// fingerprint a want names are kept: a collection never lets go of what a node is told to run. A build of a +// fingerprint no want names, one of another module at that fingerprint, and a failed one keep nothing this way. +func TestTheBuildsOfAWantedFingerprintAreKept(t *testing.T) { + fp := "src2:" + strings.Repeat("a", 64) + other := "src2:" + strings.Repeat("b", 64) + at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + made := func(id, module, fingerprint, reference string) Build { + return Build{ID: id, Module: module, SourceFingerprint: fingerprint, At: at, + Made: []Artifact{{Name: "files", Kind: "archive", Reference: reference}}} + } + const ( + wanted = "artifact-store://route-proxy/files/blobs/sha256:a1" + unwanted = "artifact-store://route-proxy/files/blobs/sha256:b2" + elsewise = "artifact-store://mesh-mail/files/blobs/sha256:c3" + ) + builds := []Build{made("key-a-1", "route-proxy", fp, wanted), made("key-b-1", "route-proxy", other, unwanted), + made("key-c-1", "mesh-mail", fp, elsewise), + {ID: "failed", Module: "route-proxy", SourceFingerprint: fp, Failed: "no", Made: []Artifact{{Reference: "artifact-store://f@sha256:ff"}}}} + kept := keptByWants(builds, map[string]bool{"route-proxy " + fp: true}) + if !has(kept[wanted], KeptByWant) { + t.Errorf("the archive of a wanted fingerprint is not kept: %v", kept) + } + for _, r := range []string{unwanted, elsewise, "artifact-store://f@sha256:ff"} { + if len(kept[r]) > 0 { + t.Errorf("%s is kept as wanted: %v", r, kept[r]) + } + } +} diff --git a/internal/inventory/doing_test.go b/internal/inventory/doing_test.go index b56c8239..8a932146 100644 --- a/internal/inventory/doing_test.go +++ b/internal/inventory/doing_test.go @@ -158,96 +158,6 @@ func TestWhatANodeSaidGoesWhenTheNodeDoes(t *testing.T) { } } -// "Behind" must mean not running what the mesh would send, not only "failed". -// -// novox/hq ADR 0010 names losing "did my change go out?" as the real risk of replacing a pipeline -// with a comparison. With behind meaning only failed-or-refused, that question was answerable -// exactly for the machines that broke — and for every machine that worked, the answer was silence -// whether the change had gone out or not. -func TestAMachineIsWaitingWhenWhatItWasSentIsNotWhatItShouldBe(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - anchor, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if _, err := inv.AddNode(ctx, "laptop"); err != nil { - t.Fatal(err) - } - - // Never sent anything: waiting, and said differently. Nobody has ever asked it to be - // anything, which is not the same as it being out of date. - waiting, err := inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"}) - if err != nil { - t.Fatal(err) - } - if len(waiting) != 2 { - t.Fatalf("machines that were never sent anything are not waiting: %+v", waiting) - } - for _, m := range waiting { - if !m.Never { - t.Fatalf("%s was never sent anything and does not say so: %+v", m.Node, m) - } - } - - // Sent what it should be: not waiting. - if err := inv.RecordSent(ctx, anchor.ID, "aaa", nil); err != nil { - t.Fatal(err) - } - waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"}) - if err != nil { - t.Fatal(err) - } - if len(waiting) != 1 || waiting[0].Node != "laptop" { - t.Fatalf("a machine sent exactly what it should be is still waiting: %+v", waiting) - } - - // The declaration changes: waiting again, and no longer "never". - waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "ccc", "laptop": "bbb"}) - if err != nil { - t.Fatal(err) - } - var found bool - for _, m := range waiting { - if m.Node != "anchor" { - continue - } - found = true - if m.Never { - t.Fatal("a machine that has been sent something is reported as never told") - } - if m.Sent != "aaa" { - t.Fatalf("what it was last sent was lost: %+v", m) - } - } - if !found { - t.Fatal("a machine whose declaration changed since it was sent is not waiting") - } -} - -// A machine nobody worked out is not reported as waiting: saying so would invent a comparison. -func TestAMachineWithNothingComputedForItIsNotWaiting(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - node, err := inv.AddNode(ctx, "unresolvable") - if err != nil { - t.Fatal(err) - } - // It has been sent something before, which is what makes this the case the guard is for: a - // machine with a digest and nothing computed for it would compare against the empty string - // and look out of date, when the truth is that nobody worked out what it should be. - if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time", nil); err != nil { - t.Fatal(err) - } - waiting, err := inv.Waiting(ctx, map[string]string{}) - if err != nil { - t.Fatal(err) - } - if len(waiting) != 0 { - t.Fatalf("a machine the caller could not work out was reported as waiting: %+v", waiting) - } -} - // **A failure that repeats is told apart from one that just happened.** A node re-applies on a // steady interval and reports each time, so a resource that will never apply arrives as the same // report over and over — one row, replaced, "failed" at a fresh time — and nothing distinguished diff --git a/internal/inventory/durations.go b/internal/inventory/durations.go deleted file mode 100644 index 5611cef8..00000000 --- a/internal/inventory/durations.go +++ /dev/null @@ -1,134 +0,0 @@ -package inventory - -import ( - "context" - "time" -) - -// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066. - -// The kinds of duration recorded. -const ( - DurationApply = "apply" - DurationPlanTier = "plan-tier" - DurationBuild = "build" - // DurationWalkPhase is one phase of an ended walk, per class (novox/hq ADR 0282 decision 6): subject - // "/", and "/total" for its merge to every machine running it. - DurationWalkPhase = "walk-phase" - // DurationBusy is a machine's wait for a busy container (novox/hq ADR 0335): from its first report of a - // send, which said a container waits because it is busy, to the report that says it replaced — the same - // send's ref as its `apply` row, whose detail is ApplyBusy. Its own measure, never in the apply's. - DurationBusy = "busy-wait" -) - -// ApplyBusy is the detail of an `apply` duration whose report said a container waits because it is busy: the -// machine answered, and is not yet running what it was sent (novox/hq ADR 0335). -const ApplyBusy = "busy" - -// DurationKinds are every kind, in the order `durations` shows them. -var DurationKinds = []string{DurationApply, DurationPlanTier, DurationBuild, DurationWalkPhase, - DurationBusy} - -// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it -// in Phase 1's first live week. -const DurationsKeptFor = 30 * 24 * time.Hour - -// Duration is one measurement. -type Duration struct { - Kind string `json:"kind"` - Subject string `json:"subject"` - Node string `json:"node,omitempty"` - Ref string `json:"ref"` - Started time.Time `json:"started"` - Took time.Duration `json:"took"` - Detail string `json:"detail,omitempty"` -} - -// RecordDuration keeps one measurement, once: the same thing measured again is not a second row. -func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error { - if d.Took < 0 { - return nil // a clock that went back measures nothing - } - _, err := i.store.Pool().Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`, - d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail) - return err -} - -// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send -// to the first report of it. A report of anything else, or of a send already measured, measures -// nothing — a machine reconciling reports the same declaration every few minutes. -func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error { - if declared == "" { - return nil - } - _, err := i.store.Pool().Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'), - n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4 - from node n - where n.name = $2 and n.sent = $3 and n.sent_at is not null - on conflict do nothing`, - DurationApply, node, declared, outcome) - return err -} - -// RecordBusyDuration measures a machine's wait for a busy container, at the report that says it replaced -// (novox/hq ADR 0335): from the end of its `apply` duration of the send it was last sent, which said it waited, to -// now. Nothing when that send's first report did not say a wait, and once per send. -func (i *Inventory) RecordBusyDuration(ctx context.Context, node, declared string) error { - if declared == "" { - return nil - } - _, err := i.store.Pool().Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - select $1, d.subject, d.node, d.ref, d.started + d.took_ms * interval '1 millisecond', - greatest(0, (extract(epoch from (now() - (d.started + d.took_ms * interval '1 millisecond'))) * 1000)::bigint), '' - from duration d join node n on n.name = d.node - where d.kind = $2 and d.subject = $3 and d.detail = $5 and n.sent = $4 and n.sent_at is not null - and d.ref = n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US') - on conflict do nothing`, - DurationBusy, DurationApply, node, declared, ApplyBusy) - return err -} - -// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty. -func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) { - rows, err := i.store.Pool().Query(ctx, - `select kind, subject, node, ref, started, took_ms, detail from duration - where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Duration - for rows.Next() { - var d Duration - var ms int64 - if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil { - return nil, err - } - d.Took = time.Duration(ms) * time.Millisecond - out = append(out, d) - } - return out, rows.Err() -} - -// WalkPhasesKept says whether any phase of a walk is kept as a walk-phase duration, and whether its total is. -func (i *Inventory) WalkPhasesKept(ctx context.Context, plan string) (anyKept, totalKept bool, err error) { - err = i.store.Pool().QueryRow(ctx, - `select count(*) > 0, count(*) filter (where ref = $2) > 0 from duration where kind = $1 and ref like $3`, - DurationWalkPhase, plan+"/total", plan+"/%").Scan(&anyKept, &totalKept) - return anyKept, totalKept, err -} - -// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many. -func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) { - tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`, - time.Now().Add(-DurationsKeptFor)) - if err != nil { - return 0, err - } - return tag.RowsAffected(), nil -} diff --git a/internal/inventory/durations_test.go b/internal/inventory/durations_test.go deleted file mode 100644 index 639866dc..00000000 --- a/internal/inventory/durations_test.go +++ /dev/null @@ -1,80 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a -// send measured at its first report and not again at every reconcile that repeats it; a send of -// something else measures nothing; a new send is a new measurement. -func TestAnApplyIsMeasuredOncePerSend(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - node, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil { - t.Fatal(err) - } - for range 3 { // the report, then two reconciles saying the same - if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil { - t.Fatal(err) - } - ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour)) - if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 { - t.Fatalf("%v %+v", err, ds) - } - time.Sleep(10 * time.Millisecond) - if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil { - t.Fatal(err) - } - if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil { - t.Fatal(err) - } - if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 { - t.Fatalf("a second send was not measured: %+v", ds) - } -} - -// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then. -func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(), - State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}}, - Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - p.State = PlanRolling // the same tier, saved again - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 { - t.Fatalf("a tier not left was measured: %+v", ds) - } - p.Tier = 1 - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - p.State = PlanDone - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - if err := inv.SavePlan(ctx, &p); err != nil { // saved again once ended: nothing more to measure - t.Fatal(err) - } - ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)) - if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" || - ds[1].Ref != "plan-1/tier-1" { - t.Fatalf("%v %+v", err, ds) - } -} diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index aac487fd..e9f6e985 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -208,3 +208,11 @@ func (i *Inventory) PlanValuesForTest(t *testing.T) map[string][]byte { } return out } + +// OnTheBusChainForTest makes an inventory opened from now on on the test's store take the chain from its opener (the bus), not +// the chain in memory the test made for the store: for a test that runs a verb in this process as the controller's child +// process runs it, over the bus the serving controller keeps the chain on. +func OnTheBusChainForTest(t *testing.T) { + t.Helper() + testChains.Delete(os.Getenv(store.Variable(Name))) +} diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go deleted file mode 100644 index f83e09c7..00000000 --- a/internal/inventory/gate.go +++ /dev/null @@ -1,335 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/jackc/pgx/v5" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// The gate's verdicts (novox/hq ADR 0236, to-be 45 §8): what a build did on its first machine, and, -// for one that failed there, how it was put back. One row per build, written by the plan that rolled it -// out, under the lease. - -// The gate's verdicts and a failed build's rollback. -const ( - GatePassed = "passed" - GateFailed = "failed" - // GateSuperseded is a judging ended by a later send to the judged machine that moved the module to - // another build (novox/hq issue 352): no verdict on the build, and nothing put back. - GateSuperseded = "superseded" - - RollingBack = "rolling-back" - RolledBack = "rolled-back" - NotRolledBack = "not-rolled-back" -) - -// GateVerdict is one build's verdict at its gate. -type GateVerdict struct { - Build string - Module string - Commit string - Previous string - Plan string - Machines []string - Verdict string - Rollback string - Why string - Component string - // JudgingFrom is when the first machine reported the build applied and the judging began. - JudgingFrom *time.Time - JudgedAt time.Time - Epoch uint64 -} - -// RecordGate writes a build's verdict. A build that passed on one machine may still fail on the next one -// judged; **a failed build's row is written once**: any later verdict for it is refused with ErrGateKept, which is what keeps a rollback to one per build — the row -// is written before the rollback's send, and a controller replaced in between finds it. -func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error { - epoch, err := i.actingEpoch(ctx) - if err != nil { - return fmt.Errorf("the gate's verdict on %s is not written: %w", v.Build, err) - } - if v.Machines == nil { - v.Machines = []string{} - } - tag, err := i.store.Pool().Exec(ctx, - `insert into build_gate (build, module, commit_hash, previous, plan, machines, verdict, rollback, why, - component, judging_from, judged_at, epoch) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12) - on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback, - why = excluded.why, previous = excluded.previous, machines = excluded.machines, - judged_at = now(), epoch = excluded.epoch - where build_gate.verdict = 'passed'`, - v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component, - v.JudgingFrom, epoch) - if err != nil { - return err - } - if tag.RowsAffected() == 0 { - return fmt.Errorf("%w: %s", ErrGateKept, v.Build) - } - return nil -} - -// ErrGateKept is a verdict already kept for the build, which is not written over. -var ErrGateKept = errors.New("this build's verdict at its gate is already kept") - -// SetRollback records how a failed build's rollback went. -func (i *Inventory) SetRollback(ctx context.Context, build, rollback, why string) error { - if _, err := i.actingEpoch(ctx); err != nil { - return err - } - _, err := i.store.Pool().Exec(ctx, - `update build_gate set rollback = $2, why = $3, judged_at = now() where build = $1 and verdict = 'failed'`, - build, rollback, why) - return err -} - -// GateOf is a build's verdict, and whether it has one. -func (i *Inventory) GateOf(ctx context.Context, build string) (GateVerdict, bool, error) { - rows, err := i.store.Pool().Query(ctx, gateSelect+` where build = $1`, build) - if err != nil { - return GateVerdict{}, false, err - } - list, err := scanGates(rows) - if err != nil || len(list) == 0 { - return GateVerdict{}, false, err - } - return list[0], true, nil -} - -// GateFailed is whether a build failed its gate: one the mesh never registers or sends again on its own. -func (i *Inventory) GateFailed(ctx context.Context, build string) (bool, error) { - v, found, err := i.GateOf(ctx, build) - return found && v.Verdict == GateFailed, err -} - -// LatestGates is the newest verdict of every module that has one: what the gate probe (DG) reads. -func (i *Inventory) LatestGates(ctx context.Context) ([]GateVerdict, error) { - rows, err := i.store.Pool().Query(ctx, `select distinct on (module) build, module, commit_hash, previous, plan, - machines, verdict, rollback, why, component, judging_from, judged_at, coalesce(epoch, 0) - from build_gate order by module, judged_at desc`) - if err != nil { - return nil, err - } - return scanGates(rows) -} - -// Gates is the newest verdicts, newest first: what `plans gates` lists. -func (i *Inventory) Gates(ctx context.Context, limit int) ([]GateVerdict, error) { - rows, err := i.store.Pool().Query(ctx, gateSelect+` order by judged_at desc limit $1`, limit) - if err != nil { - return nil, err - } - return scanGates(rows) -} - -const gateSelect = `select build, module, commit_hash, previous, plan, machines, verdict, rollback, why, component, - judging_from, judged_at, coalesce(epoch, 0) from build_gate` - -func scanGates(rows pgx.Rows) ([]GateVerdict, error) { - defer rows.Close() - var out []GateVerdict - for rows.Next() { - var v GateVerdict - var epoch int64 - if err := rows.Scan(&v.Build, &v.Module, &v.Commit, &v.Previous, &v.Plan, &v.Machines, &v.Verdict, - &v.Rollback, &v.Why, &v.Component, &v.JudgingFrom, &v.JudgedAt, &epoch); err != nil { - return nil, err - } - v.Epoch = uint64(epoch) - out = append(out, v) - } - return out, rows.Err() -} - -// PreviousBuild is the build a module goes back to when a build of it fails its gate: the newest build -// that worked, made from the commit the first machine ran before, asked before the failed one, and not -// itself failed at a gate. Among the builds whose artifacts the mesh keeps (KeptBuilds): an older one -// may already be gone from the artifact store. False when there is none to go back to. -func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, failed Build) (Build, bool, error) { - builds, err := i.Builds(ctx, module, 50) - if err != nil { - return Build{}, false, err - } - kept := 0 - for _, b := range builds { - if !b.Worked() { - continue - } - kept++ - if kept > KeptBuilds { - break - } - if b.ID == failed.ID || (commit != "" && b.Commit != commit) { - continue - } - // Only a build of the repository the failed build was made from — the module's, since its take-in - // registered it (novox/hq ADR 0266): a build of the module's name from another repository is recorded - // and was never registered, and putting it back would register it now. - if failed.Repository != "" && !sameRepositoryAs(b.Repository, failed.Repository) { - continue - } - if !failed.AskedOrAt().IsZero() && !b.AskedOrAt().Before(failed.AskedOrAt()) { - continue - } - if bad, err := i.GateFailed(ctx, b.ID); err != nil { - return Build{}, false, err - } else if bad { - continue - } - manifest, err := i.manifestOf(ctx, b.ID) - if err != nil { - return Build{}, false, err - } - if len(manifest) == 0 || string(manifest) == "null" { - continue - } - b.Manifest = manifest - return b, true, nil - } - return Build{}, false, nil -} - -// sameRepositoryAs says two recorded repositories are one, however their case or `.git` is spelled. -func sameRepositoryAs(a, b string) bool { - trim := func(s string) string { - return strings.TrimSuffix(strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/"), ".git") - } - return trim(a) == trim(b) -} - -// RestoreModule puts a module's registered build back to an earlier one: its manifest, the commit it -// was built from, and when it was asked — as now, so the build that failed its gate, asked before, can -// never register over it again (issue 219's order). The source's head is left where the merge moved it: -// the module IS behind its source, and `status` says so. -func (i *Inventory) RestoreModule(ctx context.Context, b Build) error { - if _, err := i.actingEpoch(ctx); err != nil { - return fmt.Errorf("%s is not put back: %w", b.Module, err) - } - m, err := catalogue.ParseManifest(b.Manifest) - if err != nil { - return fmt.Errorf("%s's build %s is not a manifest the mesh can register again: %w", b.Module, b.ID, err) - } - raw, err := json.Marshal(m) - if err != nil { - return err - } - tag, err := i.store.Pool().Exec(ctx, - `update module set manifest = $2, version = nullif($3, ''), built_from = nullif($4, ''), - built_asked = now(), registered = now() - where name = $1`, b.Module, raw, m.Version, b.Commit) - if err != nil { - return err - } - if tag.RowsAffected() == 0 { - return fmt.Errorf("%w: %s", ErrNoSuchModule, b.Module) - } - return nil -} - -// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a -// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told -// from one that does (the bus's planned step, ADR 0236). -func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainBuildFingerprints(ctx, c, module) -} - -// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint. -func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainFingerprints(ctx, c) -} - -// PassedCommits is, per module, the commits a build of which passed its gate on some machine. -func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bool, error) { - rows, err := i.store.Pool().Query(ctx, `select module, commit_hash from build_gate where verdict = 'passed'`) - if err != nil { - return nil, err - } - defer rows.Close() - out := map[string]map[string]bool{} - for rows.Next() { - var module, commit string - if err := rows.Scan(&module, &commit); err != nil { - return nil, err - } - if out[module] == nil { - out[module] = map[string]bool{} - } - out[module][commit] = true - } - return out, rows.Err() -} - -// BuildOf is the id of the newest successful build of a module from a commit; empty when none is -// recorded (a manifest handed over by hand). -func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { - c, err := i.theChain(ctx) - if err != nil { - return "", err - } - return i.chainBuildOf(ctx, c, module, commit) -} - -// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the -// newest successful build from it, with the artifacts of the build standing for it when its source was -// unchanged (issue 280) — what a machine last sent that build runs. False when no such build, or none -// with a manifest, is kept (novox/hq issue 295: a recorded module is composed at the build its machine -// runs until a person's push moves it). -func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (catalogue.Manifest, bool, error) { - if commit == "" { - return catalogue.Manifest{}, false, nil - } - id, raw, found, err := i.newestManifestAt(ctx, module, commit) - if err != nil { - return catalogue.Manifest{}, false, err - } - if !found { - return catalogue.Manifest{}, false, nil - } - if stands, same, err := i.StandingBuild(ctx, module, id); err != nil { - return catalogue.Manifest{}, false, err - } else if stands != "" && stands != id && len(same) > 0 { - raw = same - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - return catalogue.Manifest{}, false, fmt.Errorf("%s's build from %s is not a manifest the mesh can compose: %w", - module, commit, err) - } - return m, true, nil -} - -// manifestOf is the manifest a build was recorded with; nil when it kept none. -func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainManifestOf(ctx, c, id) -} - -// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest: -// its id and manifest; false when there is none. -func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return "", nil, false, err - } - return i.chainManifestBuild(ctx, c, module, commit) -} diff --git a/internal/inventory/given.go b/internal/inventory/given.go index fd930fea..81ab99ef 100644 --- a/internal/inventory/given.go +++ b/internal/inventory/given.go @@ -79,18 +79,18 @@ type GivenReplaced struct { } // ReplaceGivenAfterStart replaces every given value on a machine whose module has now started well -// under the mesh on it (novox/hq ADR 0228), and answers what it replaced; the caller sends the +// under the mesh on it (novox/hq ADR 0228), and answers what it replaced; the caller composes the // machines each names. // -// **The signal is the machine's clean report of the declaration it was last sent** — every resource -// applied, nothing failed or refused — **when that declaration was sent after the value was given**, -// so it is the start on the given value that counts, not an older one. On an adopted machine the -// module must also be taken: until then the mesh runs nothing of it, and nothing has started under -// the mesh. Each row is claimed by clearing its mark before it is remade, so two reports arriving -// together replace a value once; a remake that fails puts the mark back, and the next good report -// tries again. -func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared string) ([]GivenReplaced, error) { - if declared == "" { +// **The signal is the machine's have of the module being healthy** (ADR 0539 §9): started names, per +// module, when the node-engine wrote a have of it that is running and healthy at its composition. +// A value counts when that is **after it was given**, so it is the start on the given value that +// counts, not an older one. On an adopted machine the module must also be taken: until then the mesh +// runs nothing of it, and nothing has started under the mesh. Each row is claimed by clearing its mark +// before it is remade, so two haves arriving together replace a value once; a remake that fails puts +// the mark back, and the next healthy have tries again. +func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node string, started map[string]time.Time) ([]GivenReplaced, error) { + if len(started) == 0 { return nil, nil } // Only a module assigned there, from the assignments bucket. @@ -106,10 +106,10 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s `select s.node, s.module, s.name, s.replace_after_start from module_secret s join node n on n.id = s.node - where n.name = $1 and n.sent = $2 and n.sent_at > s.replace_after_start + where n.name = $1 and s.origin = 'accepted' and s.replace_after_start is not null and (not n.adopted or exists (select 1 from taken t where t.node = s.node and t.module = s.module)) - order by s.module, s.name`, node, declared) + order by s.module, s.name`, node) if err != nil { return nil, err } @@ -125,7 +125,7 @@ func (i *Inventory) ReplaceGivenAfterStart(ctx context.Context, node, declared s rows.Close() return nil, err } - if !assignedHere[d.module] { + if !assignedHere[d.module] || !started[d.module].After(d.given) { continue } dues = append(dues, d) diff --git a/internal/inventory/given_test.go b/internal/inventory/given_test.go index ff234271..cf7d9ebf 100644 --- a/internal/inventory/given_test.go +++ b/internal/inventory/given_test.go @@ -45,17 +45,9 @@ func assign(t *testing.T, inv *Inventory, ctx context.Context, node, module stri } } -// sent records a declaration sent to a machine now, as a push does, and answers its digest. -func sent(t *testing.T, inv *Inventory, ctx context.Context, node, digest string) string { - t.Helper() - n, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, n.ID, digest, nil); err != nil { - t.Fatal(err) - } - return digest +// healthy is a module's have, running and healthy, written a while from now: the first good start the store asks about. +func healthy(module string, from time.Duration) map[string]time.Time { + return map[string]time.Time{module: time.Now().Add(from)} } func originOf(t *testing.T, inv *Inventory, ctx context.Context, node, module, name string) string { @@ -68,22 +60,21 @@ func originOf(t *testing.T, inv *Inventory, ctx context.Context, node, module, n } // **A given value is replaced once, after the first good start on it, and never again** (ADR 0228): -// not on a report of a declaration sent before it was given, not on a report of a declaration the -// mesh has moved past, and not a second time. +// not on a start before it was given, and not a second time. func TestAGivenValueIsReplacedAfterTheFirstGoodStartAndNeverAgain(t *testing.T) { inv, ctx := aModuleWithGivenSecrets(t) - before := sent(t, inv, ctx, "consumer", "declaration-before") + before := healthy("letta", -time.Hour) marked, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "typed-by-a-person") if err != nil || !marked { t.Fatalf("a given value for a secret the mesh may make is marked to be replaced: %v %v", marked, err) } - // The machine's report of what it was sent before the value was given is not a start on it. + // A start before the value was given is not a start on it. if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 { t.Fatalf("a start before the value was given replaced it: %+v %v", got, err) } - carrying := sent(t, inv, ctx, "consumer", "declaration-carrying-it") - // A report about a declaration other than the one last sent says nothing about this one. + carrying := healthy("letta", time.Minute) + // The same early start again says nothing about this value. if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", before); err != nil || len(got) != 0 { t.Fatalf("a report of an older declaration replaced it: %+v %v", got, err) } @@ -106,7 +97,7 @@ func TestAGivenValueIsReplacedAfterTheFirstGoodStartAndNeverAgain(t *testing.T) if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", carrying); err != nil || len(again) != 0 { t.Fatalf("the same start replaced it twice: %+v %v", again, err) } - next := sent(t, inv, ctx, "consumer", "declaration-after") + next := healthy("letta", 2*time.Minute) if again, err := inv.ReplaceGivenAfterStart(ctx, "consumer", next); err != nil || len(again) != 0 { t.Fatalf("a later start replaced the mesh's own value: %+v %v", again, err) } @@ -126,7 +117,7 @@ func TestWhatIsNeverReplacedAfterAStart(t *testing.T) { if err := inv.AcceptSecretForModule(ctx, "consumer", "letta", "broker", "issued-by-the-mesh"); err != nil { t.Fatal(err) } - declared := sent(t, inv, ctx, "consumer", "declaration") + declared := healthy("letta", time.Minute) if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 { t.Fatalf("a value that stays as given was replaced: %+v %v", got, err) } @@ -157,14 +148,14 @@ func TestAGivenValueWaitsForTheTakeOnAnAdoptedMachine(t *testing.T) { if _, err := inv.AcceptGivenSecret(ctx, "consumer", "letta", "server-password", "the-found-one"); err != nil { t.Fatal(err) } - held := sent(t, inv, ctx, "consumer", "declaration-holding-it") + held := healthy("letta", time.Minute) if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", held); err != nil || len(got) != 0 { t.Fatalf("a module held as found, not yet taken, had its given value replaced: %+v %v", got, err) } if err := inv.Take(ctx, "consumer", "letta"); err != nil { t.Fatal(err) } - taken := sent(t, inv, ctx, "consumer", "declaration-taking-it") + taken := healthy("letta", time.Minute) got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", taken) if err != nil || len(got) != 1 { t.Fatalf("the first good start after the take did not replace the given value: %+v %v", got, err) @@ -177,7 +168,7 @@ func TestAGivenValueWaitsForTheTakeOnAnAdoptedMachine(t *testing.T) { if err := inv.Unassign(ctx, "consumer", "letta"); err != nil { t.Fatal(err) } - gone := sent(t, inv, ctx, "consumer", "declaration-without-it") + gone := healthy("letta", time.Minute) if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", gone); err != nil || len(got) != 0 { t.Fatalf("a module no longer assigned had its given value replaced: %+v %v", got, err) } @@ -196,7 +187,7 @@ func TestADefinitionThatNowSaysOutsideKeepsTheGivenValue(t *testing.T) { if err := inv.RegisterModule(ctx, m, Source{}); err != nil { t.Fatal(err) } - declared := sent(t, inv, ctx, "consumer", "declaration") + declared := healthy("letta", time.Minute) if got, err := inv.ReplaceGivenAfterStart(ctx, "consumer", declared); err != nil || len(got) != 0 { t.Fatalf("a value the definition now says is an outside party's was replaced: %+v %v", got, err) } diff --git a/internal/inventory/heartbeat_gap_gone_test.go b/internal/inventory/heartbeat_gap_gone_test.go deleted file mode 100644 index 1fcce32c..00000000 --- a/internal/inventory/heartbeat_gap_gone_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package inventory - -import "testing" - -// 1183/AC3: the heartbeat-gap duration is no kind the mesh measures. -func TestNoHeartbeatGapIsMeasured(t *testing.T) { // 1183/AC3 - for _, k := range DurationKinds { - if k == "heartbeat-gap" { - t.Fatalf("%s is still a kind of duration: %v", k, DurationKinds) - } - } -} diff --git a/internal/inventory/heldhere_test.go b/internal/inventory/heldhere_test.go deleted file mode 100644 index fe812c12..00000000 --- a/internal/inventory/heldhere_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package inventory - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" -) - -// novox/hq issue 218: a seat held once for the mesh is granted and issued only to the holder on record; -// a node seat to every machine's claimant; a mesh seat with no holder on record as derived. -func TestOnlyTheRecordedHolderHoldsAMeshSeat(t *testing.T) { - claimed := []broker.Seat{ - {Name: "mesh-store", Scope: catalogue.ScopeMesh}, - {Name: "node-packet-filter", Scope: catalogue.ScopeNode}, - {Name: "unrecorded", Scope: catalogue.ScopeMesh}, - } - holdings := []catalogue.Held{{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "control", Module: "postgres"}} - names := func(ss []broker.Seat) (out []string) { - for _, s := range ss { - out = append(out, s.Name) - } - return - } - if got := names(heldHere(claimed, holdings, "control", "postgres")); len(got) != 3 { - t.Errorf("the holder lost a seat: %v", got) - } - got := names(heldHere(claimed, holdings, "other", "postgres")) - if len(got) != 2 || got[0] != "node-packet-filter" || got[1] != "unrecorded" { - t.Errorf("a claimant on another machine holds %v; want the node seat and the unrecorded one, not the store", got) - } -} diff --git a/internal/inventory/hold.go b/internal/inventory/hold.go deleted file mode 100644 index 272a40c7..00000000 --- a/internal/inventory/hold.go +++ /dev/null @@ -1,149 +0,0 @@ -package inventory - -import ( - "context" - "errors" - "fmt" - "slices" - "strings" - "sync" - "time" -) - -// ErrNodeBusy is a node another act kept holding for longer than a caller waits. -var ErrNodeBusy = errors.New("another act is composing or sending that node's declaration") - -// HoldWaitFor is how long HoldNodes waits for a node another act holds, and HoldPoll how often it -// looks again. Variables so a test need not wait minutes. -var ( - HoldWaitFor = 2 * time.Minute - HoldPoll = 250 * time.Millisecond -) - -// HoldNodes serialises composing and sending a declaration per node (novox/hq ADR 0100): while -// one caller holds a node, another asking for it waits. Without it a push that composed a node as -// adopted could send that declaration after `converge --yes` sent the converged one, and the node -// would return to adopted with nobody having asked. -// -// Session-level advisory locks on one connection, all or none: a set not wholly free is given back -// at once, so two callers holding overlapping sets never each wait on the other. **A waiter pins -// no connection.** It looks again every HoldPoll with a connection borrowed for the look, and gives -// up after HoldWaitFor with ErrNodeBusy naming the node — so a stuck holder costs the pool one -// connection, never one per caller queued behind it. Release gives every one back, and may be -// called more than once. -func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), error) { - sorted := slices.Clone(names) - slices.Sort(sorted) - sorted = slices.Compact(sorted) - deadline := time.Now().Add(HoldWaitFor) - for { - release, busy, err := i.tryHold(ctx, sorted) - if err != nil || busy == "" { - return release, err - } - if time.Now().After(deadline) { - return nil, fmt.Errorf("%w: %s has been held for over %s — try again once it is done", - ErrNodeBusy, busy, HoldWaitFor) - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(HoldPoll): - } - } -} - -// tryHold takes every named node's lock or none, and says which node was busy when it took none. -func (i *Inventory) tryHold(ctx context.Context, sorted []string) (func(), string, error) { - conn, err := i.store.Pool().Acquire(ctx) - if err != nil { - return nil, "", err - } - var once sync.Once - release := func() { - once.Do(func() { - // Unlocking all of this session's advisory locks, then handing the connection back: a - // connection returned still holding one would hold it for whoever borrows it next. - _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`) - if err != nil { - // The session's locks die with the session: close it rather than return it. - _ = conn.Conn().Close(context.WithoutCancel(ctx)) - } - conn.Release() - }) - } - for _, name := range sorted { - var took bool - if err := conn.QueryRow(ctx, - `select pg_try_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`, - name).Scan(&took); err != nil { - release() - return nil, "", err - } - if !took { - release() - return nil, strings.TrimSpace(name), nil - } - } - return release, "", nil -} - -// ErrPlansBusy is the plans held by another act — on a machine replacing its controller, the other -// controller — for longer than a caller waits, or at all for one that does not wait. -var ErrPlansBusy = errors.New("another controller is working the plans") - -// HoldPlans makes working the plans one act at a time, across every controller on the store -// (novox/hq issue 213). A plan is read, changed and written whole; two controllers doing that at -// once — the old and the new for the moment a machine hands its controller over, or a controller -// and a person's `plans stop` — each act on what the other has not saved yet: a tier asked twice, -// an outcome written over. A session-level advisory lock on one connection, released by the -// returned function and by the session ending, so a controller that dies holding it holds nothing. -// -// wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is -// moving the plans already; and a build outcome's on the event loop, which the build records keep -// for the look to settle (novox/hq issue 1708). wait true looks again every HoldPoll for up to -// HoldWaitFor — a merge's way, which must be written. -func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) { - deadline := time.Now().Add(HoldWaitFor) - for { - release, took, err := i.tryLock(ctx, "mesh-plans") - if err != nil || took { - return release, err - } - if !wait || time.Now().After(deadline) { - return nil, ErrPlansBusy - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(HoldPoll): - } - } -} - -// tryLock takes one named advisory lock on a connection of its own, or gives the connection back. -func (i *Inventory) tryLock(ctx context.Context, key string) (func(), bool, error) { - conn, err := i.store.Pool().Acquire(ctx) - if err != nil { - return nil, false, err - } - var once sync.Once - release := func() { - once.Do(func() { - if _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`); err != nil { - _ = conn.Conn().Close(context.WithoutCancel(ctx)) - } - conn.Release() - }) - } - var took bool - if err := conn.QueryRow(ctx, `select pg_try_advisory_lock(hashtext($1)::bigint)`, key).Scan(&took); err != nil { - release() - return nil, false, err - } - if !took { - release() - return nil, false, nil - } - return release, true, nil -} diff --git a/internal/inventory/hold_plans_test.go b/internal/inventory/hold_plans_test.go deleted file mode 100644 index daa4744a..00000000 --- a/internal/inventory/hold_plans_test.go +++ /dev/null @@ -1,40 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "testing" - "time" -) - -// novox/hq issue 213: while a machine hands its controller over from the container to the process, -// two controllers run on one store for a moment. Working the plans is one act at a time across them. -func TestThePlansAreWorkedByOneControllerAtATime(t *testing.T) { - first := ForTest(t) - // A second controller: its own connections to the same store. - second, err := Open(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(second.Close) - - release, err := first.HoldPlans(t.Context(), false) - if err != nil { - t.Fatalf("the plans could not be held when nobody held them: %v", err) - } - t.Cleanup(release) // a pool closing waits for a connection still held; release is idempotent - if _, err := second.HoldPlans(t.Context(), false); !errors.Is(err, ErrPlansBusy) { - t.Fatalf("a second controller held the plans while the first did: %v", err) - } - // A waiter gets them once they are let go. - was := HoldPoll - HoldPoll = 10 * time.Millisecond - defer func() { HoldPoll = was }() - go func() { time.Sleep(50 * time.Millisecond); release() }() - again, err := second.HoldPlans(t.Context(), true) - if err != nil { - t.Fatalf("a waiting controller never got the plans once they were let go: %v", err) - } - again() -} diff --git a/internal/inventory/hold_test.go b/internal/inventory/hold_test.go deleted file mode 100644 index 19cc4132..00000000 --- a/internal/inventory/hold_test.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "strings" - "testing" - "time" -) - -// Composing and sending one node's declaration is serialised: a second holder waits for the first. -func TestHoldingANodeMakesTheNextHolderWait(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - release, err := inv.HoldNodes(ctx, []string{"anchor", "laptop"}) - if err != nil { - t.Fatal(err) - } - got := make(chan func(), 1) - go func() { - second, err := inv.HoldNodes(ctx, []string{"laptop"}) - if err != nil { - t.Error(err) - got <- func() {} - return - } - got <- second - }() - select { - case <-got: - t.Fatal("a node held by one caller was held by another at the same time") - case <-time.After(300 * time.Millisecond): - } - // Another node is not held up. - other, err := inv.HoldNodes(ctx, []string{"joiner"}) - if err != nil { - t.Fatal(err) - } - other() - release() - select { - case second := <-got: - second() - case <-time.After(5 * time.Second): - t.Fatal("releasing the node did not let the next holder in") - } -} - -// A waiter pins no pool connection while it waits, and gives up after a bounded wait saying which -// node is busy. -func TestAWaiterPinsNoConnectionAndGivesUp(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - release, err := inv.HoldNodes(ctx, []string{"anchor"}) - if err != nil { - t.Fatal(err) - } - defer release() - savedWait, savedPoll := HoldWaitFor, HoldPoll - HoldWaitFor, HoldPoll = 1500*time.Millisecond, 50*time.Millisecond - defer func() { HoldWaitFor, HoldPoll = savedWait, savedPoll }() - - pool := inv.store.Pool() - base := pool.Stat().AcquiredConns() - const waiters = 3 - done := make(chan error, waiters) - for range waiters { - go func() { - _, err := inv.HoldNodes(ctx, []string{"anchor"}) - done <- err - }() - } - // While they wait, the pool lends nothing to them for longer than a look: **pinned is held throughout**, - // so at some moment of the wait fewer than all of them hold one. Asked of the least seen over many - // looks, not of the most: a look that lands while every waiter is mid-query is a busy machine, not a - // pinned connection, and on a loaded one (the suite's packages at once, the race detector) it did land. - least := waiters - for range 100 { - time.Sleep(10 * time.Millisecond) - if n := int(pool.Stat().AcquiredConns() - base); n < least { - least = n - } - } - if least >= waiters { - t.Fatalf("%d waiters held %d connections at every look across their wait", waiters, least) - } - for range waiters { - if err := <-done; !errors.Is(err, ErrNodeBusy) || !strings.Contains(err.Error(), "anchor") { - t.Fatalf("a waiter did not give up naming the busy node: %v", err) - } - } -} diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 0014e244..643c4362 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -37,14 +37,15 @@ type Inventory struct { // chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain). chain *chain.Chain state *chainState - // recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store. - recorded func(context.Context, Duration) error // throwaway is set on an inventory raised for a moment beside the real mesh (the merge gate's): nothing it // composes is to be written to the store or the bus. throwaway bool // keptLately is how long a build's artifacts are kept after it is recorded (KeptLately, novox/hq issue 1933); zero // is KeptLately, negative none (ForTest: the tests of the keep set's other rules record their builds a moment apart). keptLately time.Duration + // wanted is every fingerprint a want names, as ` ` (novox/hq ADR 0539 clause 1; SetWanted): the + // builds of each are kept, so a collection never lets go of what a node is told to run. Nil names none. + wanted func(ctx context.Context) (map[string]bool, error) } // lately is how long a build's artifacts are kept after it is recorded, whatever else the records say. @@ -73,7 +74,6 @@ func Open(ctx context.Context) (*Inventory, error) { } func (i *Inventory) Close() { - i.stopViews() i.store.Close() } @@ -906,27 +906,6 @@ type BusyWait struct { // BusyWaiting is the state of a container left running because it said busy (link.BusyWaiting). const BusyWaiting = "waiting" -// WaitingFor is the container of a module that waits because it said busy, and whether one does: a -// resource whose id is `.`. -func WaitingFor(busy []BusyWait, module string) (BusyWait, bool) { - for _, b := range busy { - if b.State == BusyWaiting && strings.HasPrefix(b.ID, module+".") { - return b, true - } - } - return BusyWait{}, false -} - -// Waiting says whether any container waits because it said busy. -func (d Doing) Waiting() bool { - for _, b := range d.Busy { - if b.State == BusyWaiting { - return true - } - } - return false -} - // FailedResource is one thing a node could not do. type FailedResource struct { ID string `json:"id"` @@ -1154,12 +1133,15 @@ type Reported struct { Outcome string // At is when it last reported; nil if it never has. At *time.Time - // Sent is when the current declaration went to it; nil if nothing ever did. + // Sent is not read: nothing is sent (ADR 0539 §5). Always nil. The columns that recorded a send stay in the store for the + // controller of before; C6 never reads them (deferred drop, issue 1945 note 8). Sent *time.Time - // Current is whether the last report names the declaration last sent — the machine has - // acted on the current words, not merely spoken after they were written. False also covers - // a machine that has not said which, which is every host from before reports carried it. + // Current is whether the machine has everything the assigner wants of it, read from its haves beside the wants + // (cmd/mesh-controller lastReports); false here, as this reads neither. Current bool + // CurrentUnknown says Current could not be read (the bus holding the haves was not reachable): it is not "not current", + // it is not known. + CurrentUnknown bool // Declared is the digest of the declaration the last report was about, and ReportedSequence that // declaration's sequence as the report claimed it (zero from an engine that claims none): what a // gate holds against the send it made, rather than against the send made last (novox/hq issue 352). @@ -1169,67 +1151,12 @@ type Reported struct { Busy []BusyWait } -// SentDeclaration is what one send carried to a machine, as a gate keeps it: the declaration's digest and -// its sequence (novox/hq issue 352). A report about this declaration, or about one sequenced after it, is a -// report on what the gate sent — whatever the machine was sent since. -// -// At is when the controller sent it to that machine: the node's own send time, which is where that machine's -// `apply` duration starts. A walk measures its rest from it, not from its own record of the send, made only after -// every machine of the rest was sent (novox/hq issue 582). Zero for a send kept before it was recorded. -type SentDeclaration struct { - Digest string `json:"digest"` - Sequence int64 `json:"sequence,omitempty"` - At time.Time `json:"at,omitzero"` - // Provisioned is, for each module the send handed to the provisioner, the digest of the templates its plan - // named (novox/hq ADR 0349 §4, ADR 0362): the build the machine's provisioner client reports `prov.applied` on. - // The machine has applied the send of such a module only once both it and the client reported it. - Provisioned map[string]string `json:"provisioned,omitempty"` -} - -// ReportsOn says a report is about this send: the declaration itself; one the same machine was sequenced -// after it; or the declaration the machine was sent last (Current), which is this send or a later one — -// sends to a machine are made one after another. A send kept without a sequence is matched by its digest -// and by the last send alone. -func (s SentDeclaration) ReportsOn(r Reported) bool { - if r.Current || (s.Digest != "" && r.Declared == s.Digest) { - return true - } - return s.Sequence > 0 && r.ReportedSequence >= s.Sequence -} - -// SentTo is the declaration a machine was last sent, by name: its digest and sequence, and false when it -// was never sent one. -func (i *Inventory) SentTo(ctx context.Context, name string) (SentDeclaration, bool, error) { - var digest *string - var seq *int64 - var at *time.Time - var plans []byte - err := i.store.Pool().QueryRow(ctx, `select sent, sequence, sent_at, sent_provisioning from node where name = $1`, - name).Scan(&digest, &seq, &at, &plans) - if errors.Is(err, pgx.ErrNoRows) { - return SentDeclaration{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, name) - } - if err != nil || digest == nil || *digest == "" { - return SentDeclaration{}, false, err - } - s := SentDeclaration{Digest: *digest} - if seq != nil { - s.Sequence = *seq - } - if at != nil { - s.At = at.UTC() - } - if s.Provisioned, err = provisionedBy(plans); err != nil { - return SentDeclaration{}, false, fmt.Errorf("what %s was sent of the provisioner's plans: %w", name, err) - } - return s, true, nil -} - -// LastReports is every machine's last report beside when it was last sent a declaration. +// LastReports is every machine's last report: when, what it did, which declaration it was about and what it holds busy. Sent +// and Current are not kept here: nothing is sent (ADR 0539 §5), and whether a machine has what is wanted of it is read from +// its haves (cmd/mesh-controller lastReports). func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) { rows, err := i.store.Pool().Query(ctx, - `select n.name, coalesce(r.outcome, ''), r.at, n.sent_at, - coalesce(r.declared is not null and r.declared <> '' and r.declared = n.sent, false), + `select n.name, coalesce(r.outcome, ''), r.at, coalesce(r.declared, ''), coalesce(r.reported_sequence, 0), coalesce(r.busy, '[]') from node n left join node_report r on r.node = n.id order by n.name`) @@ -1241,8 +1168,7 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) { for rows.Next() { var r Reported var busy []byte - if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Sent, &r.Current, &r.Declared, &r.ReportedSequence, - &busy); err != nil { + if err := rows.Scan(&r.Node, &r.Outcome, &r.At, &r.Declared, &r.ReportedSequence, &busy); err != nil { return nil, err } if err := json.Unmarshal(busy, &r.Busy); err != nil { @@ -1253,165 +1179,6 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) { return out, rows.Err() } -// RecordSent keeps a digest of the declaration a machine was last sent, and the build of each module -// it carried. -// -// **A digest rather than the declaration.** The mesh can compute what a machine should be at any -// moment; keeping a copy would be a second account of it, able to disagree with the first. What -// cannot be recomputed is what was *actually sent*, and that is the whole difference between a -// machine that is out of date and one that has never been told. -// -// **And which build of each module** (novox/hq issue 259, ADR 0221): module name to the commit its -// build was made from. A digest cannot say whether a machine differs because a module moved to a build -// its upgrade policy holds back, or because of something a push made — a grant — and only the second -// is a push's to send to a machine it did not name. Nil records that it is not known, as for a -// declaration sent by hand. -func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds map[string]string) error { - return i.RecordSentUnder(ctx, node, digest, builds, 0) -} - -// RecordSentUnder is RecordSent for a declaration that carried an epoch (novox/hq to-be 45 §6): kept -// beside the digest, so what the mesh would send is composed with it. Zero is one that carried none. -func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, builds map[string]string, - epoch uint64) error { - var sentEpoch *int64 - if epoch > 0 { - e := int64(epoch) - sentEpoch = &e - } - if _, err := i.store.Pool().Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil { - return err - } - // What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record. - return i.recordRuns(ctx, node, builds, time.Now()) -} - -// AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue -// 275): the oldest assignment on it made after it was last sent, or — when nothing was assigned since — -// when it was last sent, or when it joined if it never was. The moment a bound on "not pushed yet" is -// read from: every change a push carries happened after the last push, and an assignment is the one -// that says when. -func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, error) { - return i.chainAwaitingSince(ctx, name) -} - -// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit -// its build was made from (novox/hq issue 259), read from the keys of the machine's assignments that name a commit — an -// assigned module and one only pulled in (ADR 0408, issue 1109). Known is false when that was not kept: a machine never sent -// anything, or last sent a declaration by hand, which makes every key of it say its commit unknown. -func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) { - var sentAt *time.Time - err = i.store.Pool().QueryRow(ctx, `select sent_at from node where name = $1`, name).Scan(&sentAt) - if errors.Is(err, pgx.ErrNoRows) { - return nil, false, nil - } - if err != nil || sentAt == nil { - return nil, false, err - } - c, err := i.theChain(ctx) - if err != nil { - return nil, false, err - } - all, err := c.Assignments.All(ctx) - if err != nil { - return nil, false, fmt.Errorf("what %s was sent cannot be read from the bus: %w", name, err) - } - builds = map[string]string{} - for _, k := range all { - a := k.Value - if a.Node != name || !a.Stands() { - continue - } - if !a.SentUnknown.IsZero() { - return nil, false, nil - } - if a.Commit != "" { - builds[a.Module] = a.Commit - } - } - return builds, true, nil -} - -// RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name -// (novox/hq issue 249): whether the machine holding the bus must go first is whether this differs -// from the list composed now. -func (i *Inventory) RecordSentBusUsers(ctx context.Context, name, digest string) error { - _, err := i.store.Pool().Exec(ctx, - `update node set sent_bus_users = $2 where name = $1`, name, digest) - return err -} - -// SentBusUsers is the digest of the bus's user list a machine was last sent, empty for none. -func (i *Inventory) SentBusUsers(ctx context.Context, name string) (string, error) { - var sent string - err := i.store.Pool().QueryRow(ctx, - `select sent_bus_users from node where name = $1`, name).Scan(&sent) - if errors.Is(err, pgx.ErrNoRows) { - return "", nil - } - return sent, err -} - -// Outstanding is the digest of the declaration a machine was last sent, by its name, and empty -// for one that has never been sent anything. -// -// **By name rather than by id**, because the caller is the serving loop and what a node puts in a -// report is its name. Asked of one machine rather than read from Waiting's sweep, because it is -// asked per message: a report names the declaration it is about, and a report about one the mesh -// has already moved past is not acted on (design 25 §3). -func (i *Inventory) Outstanding(ctx context.Context, name string) (string, error) { - var sent string - err := i.store.Pool().QueryRow(ctx, - `select coalesce(sent, '') from node where name = $1`, name).Scan(&sent) - if errors.Is(err, pgx.ErrNoRows) { - // Not an error worth carrying up: a report from a machine the mesh has no record of has - // nothing to be stale against, and whatever is wrong with it is the listener's to say. - return "", nil - } - return sent, err -} - -// Waiting is every machine whose declaration has changed since it was last sent one. -// -// The caller works out what each machine should be now, because only it can — resolution is the -// control plane's and this context holds records. What is answered here is the comparison. -// -// **A machine that has never been sent anything is waiting**, and says so differently: it is not -// out of date, it has never been told, and the remedy is the same push while the situation is not -// the same at all. -func (i *Inventory) Waiting(ctx context.Context, would map[string]string) ([]Machine, error) { - rows, err := i.store.Pool().Query(ctx, - `select name, coalesce(sent, ''), sent_at from node order by name`) - if err != nil { - return nil, err - } - defer rows.Close() - - var out []Machine - for rows.Next() { - var m Machine - var at *time.Time - if err := rows.Scan(&m.Node, &m.Sent, &at); err != nil { - return nil, err - } - m.SentAt = at - wanted, known := would[m.Node] - if !known { - // Nothing was computed for it — it resolves to nothing, or the caller did not ask. - // Silence rather than a guess: saying "waiting" about a machine nobody worked out - // would be inventing a comparison. - continue - } - if m.Sent == wanted { - continue - } - m.Never = m.Sent == "" - out = append(out, m) - } - return out, rows.Err() -} - // Machine is one machine that has not been sent what it should be. type Machine struct { Node string diff --git a/internal/inventory/older_build_test.go b/internal/inventory/older_build_test.go deleted file mode 100644 index eab509ef..00000000 --- a/internal/inventory/older_build_test.go +++ /dev/null @@ -1,126 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "errors" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// novox/hq 04-ISSUES/219: two builds of one module in flight together, the one asked first heard -// last. The newer request stood on the newer base; the older one's late outcome is recorded and is -// not what the module is. - -func postgresBuild(id string, asked time.Time, image string) Build { - b := aBuild(id, "postgres", "") - b.Asked = asked - b.Against = []string{"mesh-tools/runtime@sha256:" + id} - b.Made = []Artifact{{Name: "server", Kind: "image", Reference: "postgres@sha256:" + image}} - return b -} - -func TestAnOlderRequestFinishingLaterIsNotWhatTheModuleHolds(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) - newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) - - // The newer request finishes first, the older one last — recorded in that order. - if err := inv.RecordBuild(ctx, postgresBuild("newer", newer, "4bcd5f73")); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, postgresBuild("older", older, "0ab07fa9")); err != nil { - t.Fatal(err) - } - - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - if got := held["postgres/server"]; got != "postgres@sha256:4bcd5f73" { - t.Errorf("postgres holds %q; want the newer request's image 4bcd5f73", got) - } - against, err := inv.BuiltAgainst(ctx) - if err != nil { - t.Fatal(err) - } - if got := against["postgres"]; len(got) != 1 || got[0] != "mesh-tools/runtime@sha256:newer" { - t.Errorf("postgres stands on %v; want what the newer request stood on", got) - } - - // Both are still recorded, the late one first as what happened lately. - builds, err := inv.Builds(ctx, "postgres", 5) - if err != nil { - t.Fatal(err) - } - if len(builds) != 2 || builds[0].ID != "older" || !builds[0].Asked.Equal(older) { - t.Fatalf("both builds, newest heard first, with when they were asked: %+v", builds) - } -} - -func TestABuildWithNoKnownRequestTimeIsOrderedByWhenItWasRecorded(t *testing.T) { - // What the mesh did before it kept the request time, so a row from before still answers. - inv := fresh(t) - ctx := context.Background() - for _, id := range []string{"first", "second"} { - b := aBuild(id, "shell", "") - b.Made = []Artifact{{Name: "config", Kind: "archive", Reference: "…/" + id}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - if got := held["shell/config"]; got != "…/second" { - t.Errorf("shell holds %q; want the one recorded last", got) - } -} - -func TestARegistrationFromAnOlderRequestDoesNotReplaceANewerOne(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) - newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) - from := func(asked time.Time) Source { - return Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/postgres", - BuiltFrom: "efff5415", Asked: asked} - } - - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "fixed"}, from(newer)); err != nil { - t.Fatal(err) - } - err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "stale"}, from(older)) - if !errors.Is(err, ErrSuperseded) { - t.Fatalf("an older request's registration was not refused as superseded: %v", err) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if got := shelf["postgres"].Version; got != "fixed" { - t.Fatalf("postgres is %q; want the newer request's manifest", got) - } - - // A later request, and a manifest handed over by hand — asked when it is handed over — both - // replace it as before. - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "later"}, - from(newer.Add(time.Minute))); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "by-hand"}, Source{}); err != nil { - t.Fatal(err) - } - if shelf, _ := inv.Catalogue(ctx); shelf["postgres"].Version != "by-hand" { - t.Fatalf("postgres is %q; want the manifest handed over by hand", shelf["postgres"].Version) - } - src, err := inv.SourceOf(ctx, "postgres") - if err != nil || src.Repository != "novox/mesh-catalog" { - t.Fatalf("a hand registration erased the provenance: %+v %v", src, err) - } -} diff --git a/internal/inventory/order.go b/internal/inventory/order.go index 33834b37..9e15fa27 100644 --- a/internal/inventory/order.go +++ b/internal/inventory/order.go @@ -142,34 +142,6 @@ func scanEpochs(rows pgx.Rows) ([]Epoch, error) { return out, rows.Err() } -// SentEpoch is the epoch a machine was last sent, by its id; zero for one sent without. -func (i *Inventory) SentEpoch(ctx context.Context, id string) (uint64, error) { - var epoch *int64 - if err := i.store.Pool().QueryRow(ctx, `select sent_epoch from node where id = $1`, id).Scan(&epoch); err != nil { - return 0, fmt.Errorf("reading the epoch %s was last sent: %w", id, err) - } - if epoch == nil { - return 0, nil - } - return uint64(*epoch), nil -} - -// ReadsEpoch says a machine's node-engine said it reads an epoch in a declaration, by its id. -func (i *Inventory) ReadsEpoch(ctx context.Context, id string) (bool, error) { - var reads bool - if err := i.store.Pool().QueryRow(ctx, `select reads_epoch from node where id = $1`, id).Scan(&reads); err != nil { - return false, fmt.Errorf("reading whether %s reads an epoch: %w", id, err) - } - return reads, nil -} - -// RecordReadsEpoch keeps what a machine's latest report said of reading an epoch. Every report of a -// node-engine that orders its reports says it, so a node-engine rolled back says it no longer does. -func (i *Inventory) RecordReadsEpoch(ctx context.Context, id string, reads bool) error { - _, err := i.store.Pool().Exec(ctx, `update node set reads_epoch = $2 where id = $1`, id, reads) - return err -} - // ReportOrder is the order of an account: the declaration's epoch and sequence it is about, and the // node-engine's own report sequence. Zero in any claims none. type ReportOrder struct { diff --git a/internal/inventory/order_test.go b/internal/inventory/order_test.go deleted file mode 100644 index 910d783a..00000000 --- a/internal/inventory/order_test.go +++ /dev/null @@ -1,128 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "errors" - "testing" - "time" -) - -// Order and one writer, as the store keeps them (novox/hq to-be 45 §6). - -// **A plan is written by compare-and-set on its revision, carrying the epoch**: a write against a plan -// another writer moved since is refused, and nothing is written by a process that may not act. -func TestAPlanIsWrittenByCompareAndSetUnderTheLease(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - epoch := uint64(57) - inv.ActsUnder(func(context.Context) (uint64, error) { return epoch, nil }) - - p := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), State: PlanBuilding, - Tiers: [][]string{{"app"}}, Modules: map[string]*PlanModule{"app": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - if p.Revision != 1 || p.Epoch != 57 { - t.Fatalf("a new plan was written at revision %d, epoch %d", p.Revision, p.Epoch) - } - // Two readers of revision 1: the first write wins, the second is refused and writes nothing. - first, err := inv.PlanByID(ctx, "plan-cas") - if err != nil { - t.Fatal(err) - } - second := first - first.Note = "the newer word" - if err := inv.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - second.State = PlanFailed - if err := inv.SavePlan(ctx, &second); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a write against a plan moved since it was read was not refused: %v", err) - } - kept, _ := inv.PlanByID(ctx, "plan-cas") - if kept.State != PlanBuilding || kept.Note != "the newer word" || kept.Revision != 2 { - t.Fatalf("the refused write reached the plan: %+v", kept) - } - // The writer saves its own plan again without reading it: its revision moved with its write. - first.Tier = 0 - if err := inv.SavePlan(ctx, &first); err != nil { - t.Fatalf("a writer could not save its own plan twice: %v", err) - } - // A new plan under an id already written is refused, not laid over it. - again := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "deadbeef", Created: time.Now(), State: PlanBuilding} - if err := inv.SavePlan(ctx, &again); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a second plan under one id was written: %v", err) - } - // And a process that does not hold the lease writes nothing. - inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }) - first.Note = "from a controller that lost the lease" - if err := inv.SavePlan(ctx, &first); err == nil { - t.Fatal("a plan was written by a controller that does not hold the lease") - } - if kept, _ := inv.PlanByID(ctx, "plan-cas"); kept.Note != "the newer word" || kept.Epoch != 57 { - t.Fatalf("a controller without the lease wrote the plan: %+v", kept) - } -} - -// The epochs the mesh issued: the highest is the floor; taking one ends every earlier one nobody gave -// back as found expired, and says which; one given back says so and is not said again. -func TestTheEpochsIssuedAreKept(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - if highest, err := inv.HighestEpoch(ctx); err != nil || highest != 0 { - t.Fatalf("a mesh that issued none has %d (%v)", highest, err) - } - if _, err := inv.TookEpoch(ctx, Epoch{Epoch: 41, Instance: "a", Taken: time.Now()}); err != nil { - t.Fatal(err) - } - // a stopped renewing and said nothing; b takes the lease. - expired, err := inv.TookEpoch(ctx, Epoch{Epoch: 57, Instance: "b", Taken: time.Now()}) - if err != nil || len(expired) != 1 || expired[0].Epoch != 41 || expired[0].How != EpochExpired { - t.Fatalf("taking 57 ended %+v (%v), want 41 found expired", expired, err) - } - if err := inv.EndEpoch(ctx, 57, EpochReleased); err != nil { - t.Fatal(err) - } - expired, err = inv.TookEpoch(ctx, Epoch{Epoch: 60, Instance: "c", Taken: time.Now()}) - if err != nil || len(expired) != 0 { - t.Fatalf("a lease given back was found expired: %+v (%v)", expired, err) - } - if highest, _ := inv.HighestEpoch(ctx); highest != 60 { - t.Fatalf("the highest epoch issued is %d, want 60", highest) - } - e, found, err := inv.EpochOf(ctx, 57) - if err != nil || !found || e.Instance != "b" || e.How != EpochReleased || e.Ended == nil { - t.Fatalf("epoch 57 reads %+v (%v, %v)", e, found, err) - } - recent, err := inv.EpochsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(recent) != 3 || recent[0].Epoch != 60 || recent[0].Ended != nil { - t.Fatalf("the epochs of the last hour read %+v (%v)", recent, err) - } -} - -// What a machine was sent under, and whether it reads an epoch. -func TestTheEpochAMachineWasSentIsKept(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - node, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSentUnder(ctx, node.ID, "d1", nil, 57); err != nil { - t.Fatal(err) - } - if e, err := inv.SentEpoch(ctx, node.ID); err != nil || e != 57 { - t.Fatalf("sent under %d (%v)", e, err) - } - if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil { - t.Fatal(err) - } - if e, _ := inv.SentEpoch(ctx, node.ID); e != 0 { - t.Fatalf("a declaration sent without an epoch left %d kept", e) - } - if reads, _ := inv.ReadsEpoch(ctx, node.ID); reads { - t.Fatal("a machine that never said so reads an epoch") - } -} diff --git a/internal/inventory/plan_backups_test.go b/internal/inventory/plan_backups_test.go deleted file mode 100644 index 40a9f301..00000000 --- a/internal/inventory/plan_backups_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package inventory - -import ( - "encoding/json" - "reflect" - "strings" - "testing" - "time" -) - -// The plan's backups are the contract mesh-delivery reads from `plan-moved` (issue 1065): their JSON field names -// are fixed here, and a plan keeps them through a round trip. -func TestAPlanKeepsItsBackupsThroughItsJSONUnderTheNamesTheDeliveryReads(t *testing.T) { - at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - p := Plan{ID: "p", Backups: &PlanBackups{State: BackupFailed, NoHolder: []string{"bare"}, - Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupFailed, AskedAt: &at, Words: "no space left"}}}} - raw, err := json.Marshal(p) - if err != nil { - t.Fatal(err) - } - for _, name := range []string{`"backups":{"state":"failed","asked":[{"machine":"ace","module":"mail","state":"failed",` + - `"asked_at":"2026-10-17T12:00:00Z","words":"no space left"}],"no_holder":["bare"]}`} { - if !strings.Contains(string(raw), name) { - t.Fatalf("plan JSON %s lacks %s", raw, name) - } - } - var back Plan - if err := json.Unmarshal(raw, &back); err != nil || !reflect.DeepEqual(back.Backups, p.Backups) { - t.Fatalf("round trip: %v %+v", err, back.Backups) - } - raw, _ = json.Marshal(Plan{ID: "q"}) - if strings.Contains(string(raw), "backups") { - t.Fatalf("a plan with no backups says none: %s", raw) - } -} diff --git a/internal/inventory/plan_id_test.go b/internal/inventory/plan_id_test.go deleted file mode 100644 index 431643b2..00000000 --- a/internal/inventory/plan_id_test.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// **A plan's id is a value, never a query** (novox/hq issue 561): the id reaches PlanByID from a caller of the -// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no -// other plan answered for it. On the bus an id is a key, and one the bus takes no spelling of is refused to be kept. -func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(), - State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - for _, id := range []string{ - "x' or '1'='1", // every plan, when pasted into the query - "plan-561' or id like '%", // the plan above, by a pattern - "it's", // a quote alone: a syntax error, when pasted - } { - got, err := inv.PlanByID(ctx, id) - if err == nil { - t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID) - } - if want := "no plan " + id; err.Error() != want { - t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want) - } - } - // A plan whose id holds a quote cannot be a key of the bus: refused by name, never rewritten into another id. - quoted := p - quoted.ID, quoted.Revision = "plan-'561'", 0 - if err := inv.SavePlan(ctx, "ed); !errors.Is(err, chain.ErrBadKey) { - t.Fatalf("a plan with a quote in its id was kept, or refused for another reason: %v", err) - } - if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") { - t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent) - } -} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go deleted file mode 100644 index 90af4a0c..00000000 --- a/internal/inventory/plans.go +++ /dev/null @@ -1,765 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything -// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in -// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge -// still waits for. -type Plan struct { - ID string `json:"id"` - Repository string `json:"repository"` - // Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open - // ones of the same repository and branch. Empty for a plan from before it was kept. - Branch string `json:"branch,omitempty"` - Commit string `json:"commit"` - // Merged is when the forge made the merge this plan answers (novox/hq issue 349): the order of a - // branch's merges, which is not the order their plans were made in when one was acted on late. Zero - // for a release, and for a plan kept before it was. - Merged time.Time `json:"merged,omitzero"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - State string `json:"state"` - Tier int `json:"tier"` - Tiers [][]string `json:"tiers"` - Modules map[string]*PlanModule `json:"modules"` - Note string `json:"note,omitempty"` - // Away is the machines the walk passed because they were out of touch as a tier was sent (novox/hq ADR 0435 §1, - // issue 1161): not sent to and not waited for. Their want of the walk's builds stays for the send they get when - // they are heard again (ADR 0360 §5). Sorted. - Away []string `json:"away,omitempty"` - // TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and - // never written from here: a save measures the tier it leaves and stamps the next. What the - // watchdog of a plan's progress (S3) reads. - TierEntered time.Time `json:"tier_entered,omitempty"` - // Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a - // plan is written by compare-and-set, so a write against a plan another writer moved since is - // refused rather than laid over it. Zero is a plan never saved. SavePlan moves it. - Revision int64 `json:"revision"` - // Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none. - Epoch uint64 `json:"epoch,omitempty"` - // Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a - // gate, walked through the machines one at a time. - Release *PlanRelease `json:"release,omitempty"` - // Delivery is set on a walk that waits for its delivery's word (novox/hq ADR 0239), and on every walk and - // batch that names the merges it answers (ADR 0276): Awaits empty for a walk on the controller's own path, - // which starts when it is cut. - Delivery *PlanDelivery `json:"delivery,omitempty"` - // Commits is every repository's commit the walk carries, one per repository (novox/hq ADR 0276): a batch's - // walk can carry several. Repository and Commit above keep one of them, for a reader that knows one. Empty - // for a walk kept before it was: Repository and Commit are then the whole of it. - Commits []PlanCommit `json:"commits,omitempty"` - // Times are the moments of a walk no other field keeps (novox/hq ADR 0282 decision 6): when its batch's - // window closed, when it was cut, and its class. Nil for a walk kept before they were, whose phases before - // its build are said unknown. - Times *PlanTimes `json:"times,omitempty"` - // Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out - // from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`). - Phases *WalkPhases `json:"phases,omitempty"` - // Backups is the backups the walk takes before it asks its first tier (novox/hq ADR 0404 decisions 2 and 3): - // per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a - // walk that moves no module with an item protected by a backup, and for a walk kept before it was. - Backups *PlanBackups `json:"backups,omitempty"` - // Links are what the rollout names of the chain (novox/hq ADR 0408 §1): the builds it carried, the machine steps it has - // taken and the assignments it changed, each with the build before and after. Kept in the rollout's record beside - // the fields above; empty in the controller's store. - chain.Links - // Held are the merges this plan or batch holds (novox/hq ADR 0276, ADR 0408): what the store kept as batched merges - // with this plan's id, and the merges no plan answers that it held last. **Written by the merge operations only**: a - // save of the plan keeps what the record holds, never what this copy read, so a copy read before a merge moved cannot - // lay it back (issue 1092). - Held []chain.HeldMerge `json:"held,omitempty"` - // WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308 - // decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every - // other save. - WalkSaid string `json:"-"` -} - -// The states of a plan's backups, as a whole and each. -const ( - BackupPending = "pending" - BackupDone = "done" - BackupFailed = "failed" -) - -// PlanBackups is a walk's record of the backups it takes before it asks anything (novox/hq ADR 0404): read by -// mesh-delivery from `plan-moved`, so its JSON field names are a contract between the two repositories. -type PlanBackups struct { - // State is the whole: pending while any backup is asked or unasked and none failed, failed as soon as one - // failed, done when every one is. - State string `json:"state"` - // Asked is one entry per machine and module, sorted by machine then module. - Asked []PlanBackup `json:"asked"` - // NoHolder are the machines that run a module with an item protected by a backup and hold no node-backup: - // nothing is asked of them, and the plan says so. - NoHolder []string `json:"no_holder,omitempty"` -} - -// PlanBackup is one backup asked of one machine's backup holder for one module. -type PlanBackup struct { - Machine string `json:"machine"` - Module string `json:"module"` - // State is pending, done or failed. - State string `json:"state"` - // AskedAt is when the holder was asked; nil while it is not asked yet. - AskedAt *time.Time `json:"asked_at,omitempty"` - // TriedAt is when the controller first tried to ask, kept while the holder cannot be reached: the bound of a - // backup that is never asked counts from it. Nil once asked. - TriedAt *time.Time `json:"tried_at,omitempty"` - // Words are the backup's own: the holder's error for a failed one, "did not answer in ten minutes" for a silent one. - Words string `json:"words,omitempty"` -} - -// PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6). -type PlanTimes struct { - // WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum. - // Nil for a walk no window assembled (one merge walked alone). - WindowClosed *time.Time `json:"window_closed,omitempty"` - // Cut is when the batch became the walk. - Cut *time.Time `json:"cut,omitempty"` - // Class is the walk's module class, read at its cut: core when it moves a module on the controller's own - // path or the mesh's resolver, leaf otherwise (ADR 0282 decision 1). - Class string `json:"class,omitempty"` - // Started is when the walk was said started, and Ended when it was said ended (novox/hq ADR 0308 decision - // 1): each said once, on the save that marked it. - Started *time.Time `json:"started,omitempty"` - Ended *time.Time `json:"ended,omitempty"` -} - -// PlanCommit is one repository's commit a walk carries: the latest merge of its branch in the batch. -type PlanCommit struct { - Repository string `json:"repository"` - Branch string `json:"branch,omitempty"` - Commit string `json:"commit"` - Merged time.Time `json:"merged,omitzero"` -} - -// PlanMerge is one merge a walk or batch answers (novox/hq ADR 0276), as mesh-delivery reads it: Carried is -// the commit of its repository the walk builds and sends when that is a later merge containing it, and -// empty when it is the merge itself. -type PlanMerge struct { - Repository string `json:"repository"` - Commit string `json:"commit"` - Carried string `json:"carried,omitempty"` - // Number and Title are the merge's pull request as the forge announced it, and Moves the modules the merge - // moved when it was heard: what `plans` names a walk by. Empty where the announcement said none. - Number int `json:"number,omitempty"` - Title string `json:"title,omitempty"` - Moves []string `json:"moves,omitempty"` - // Head is the pull request's head commit the forge merged (novox/hq issue 703): the walk takes only a build a - // check of that head made. Empty where the announcement said none, and then it takes none. - Head string `json:"head,omitempty"` - // Merged is when the forge made the merge and Heard when the controller heard it (novox/hq ADR 0282): where - // its delivery time starts. Zero for a merge named before they were kept. - Merged time.Time `json:"merged,omitzero"` - Heard time.Time `json:"heard,omitzero"` -} - -// PlanBatch is a batch's window while it is one (novox/hq ADR 0276): when it closes unless another merge -// comes, when it closes at the latest, and the walk it waits behind once closed. -type PlanBatch struct { - ClosesAt time.Time `json:"closes_at"` - AtMost time.Time `json:"at_most"` - Behind string `json:"behind,omitempty"` - // BehindWhy says why the batch waits behind that walk: what the two share (issue 973). - BehindWhy string `json:"behind_why,omitempty"` - // Own says the batch holds merges on the controller's own path, whose walk waits for nobody's word: such - // a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the - // build, 2026-10-10). - Own bool `json:"own,omitempty"` - // Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they - // were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are - // the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so - // the batch's walk does not build them. - Closed bool `json:"closed,omitempty"` - Withheld []string `json:"withheld,omitempty"` - // CutAtOnce says the batch's window closed when it was made: its merge shares nothing with a batch - // assembling and has no delivery group member still to merge, so it waits for no window (novox/hq ADR 0304 - // decision 1, ADR 0306 decisions 2 and 3). A merge that joins it later opens its window again. It is not - // Closed, which is a stop's. - CutAtOnce bool `json:"cut_at_once,omitempty"` - // Waits says why a batch made for one merge waits for the window: its delivery group has a member still to - // merge, or mesh-delivery could not say whether it has (novox/hq ADR 0306 decision 2). - Waits string `json:"waits,omitempty"` - // Holds are the groups whose last member is still to merge, by head branch name, each with why in words - // (novox/hq ADR 0418 clause 3): the batch does not close by the merge window while one is held, however long the - // member takes. Cleared when every member is merged or closed; the window then runs again from that moment. - Holds map[string]string `json:"holds,omitempty"` - // HeldAt is when the last hold was cleared: the maximum is counted from it, not from the batch's making. - HeldAt time.Time `json:"held_at,omitempty"` -} - -// OwnPath says the record is a batch of merges on the controller's own path. -func (p Plan) OwnPath() bool { - return p.Delivery != nil && p.Delivery.Batch != nil && p.Delivery.Batch.Own -} - -// CommitOf is the commit of a repository the walk carries; empty when it carries none of it. -func (p Plan) CommitOf(repository string) string { - for _, c := range p.Commits { - if strings.EqualFold(c.Repository, repository) { - return c.Commit - } - } - if strings.EqualFold(p.Repository, repository) { - return p.Commit - } - return "" -} - -// CommitOn is the commit of a repository's branch the walk carries; empty when it carries none of it. A -// commit kept without its branch matches any branch. -func (p Plan) CommitOn(repository, branch string) string { - for _, c := range p.Commits { - if strings.EqualFold(c.Repository, repository) && (branch == "" || c.Branch == "" || c.Branch == branch) { - return c.Commit - } - } - if len(p.Commits) == 0 && strings.EqualFold(p.Repository, repository) && - (branch == "" || p.Branch == "" || p.Branch == branch) { - return p.Commit - } - return "" -} - -// Carried is every repository's commit the walk carries: Commits, or its one repository and commit. -func (p Plan) Carried() []PlanCommit { - if len(p.Commits) > 0 { - return p.Commits - } - if p.Repository == "" && p.Commit == "" { - return nil - } - return []PlanCommit{{Repository: p.Repository, Branch: p.Branch, Commit: p.Commit, Merged: p.Merged}} -} - -// Named is the walk as a person reads it: each repository at its commit. -func (p Plan) Named() string { - var out []string - for _, c := range p.Carried() { - out = append(out, c.Repository+" "+short(c.Commit)) - } - return strings.Join(out, ", ") -} - -// Batch says the record is a batch still being assembled or waiting to be cut, not a walk (ADR 0276). -func (p Plan) Batch() bool { return p.State == PlanAssembling || p.State == PlanQueued } - -// PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239). -type PlanDelivery struct { - // Order are the edges the plan's own order was drawn with (novox/hq ADR 0419): verb before caller, version skew, - // engine before controller. Kept on the plan because the catalogue's graph does not hold them, and the plan's gates - // (a later tier waits for the earlier to be applied) are read from them. - Order []Edge `json:"order,omitempty"` - // Awaits is who must say the walk may start: the seat whose holder owns the delivery. - Awaits string `json:"awaits"` - // Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why. - Go *time.Time `json:"go,omitempty"` - By string `json:"by,omitempty"` - Why string `json:"why,omitempty"` - // Stopped is who ended the walk, through the delivery's owner or by hand (`plans stop|close`), and StoppedWhy - // why: the walk is failed, and the delivery reads it as stopped, not as a build that failed. - Stopped string `json:"stopped,omitempty"` - StoppedWhy string `json:"stopped_why,omitempty"` - // Merges is every merge the walk or batch answers (novox/hq ADR 0276): its own commits, and the earlier - // merges of a repository its later commit contains. - Merges []PlanMerge `json:"merges,omitempty"` - // TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that - // walk's now. - TakenOverBy string `json:"taken_over_by,omitempty"` - // WalkedAgainBy names, on a walk stopped without some of its merges, the batch that walks its other merges - // again, and LeftOut the merges it left out (novox/hq issue 459, ADR 0299): a delivery of a merge left out - // ends stopped with the walk, one of a merge walked again follows that batch's walk. - WalkedAgainBy string `json:"walked_again_by,omitempty"` - LeftOut []PlanLeftOut `json:"left_out,omitempty"` - // Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running - // the stopped walk's build (novox/hq ADR 0299). - Kept []PlanKept `json:"kept,omitempty"` - // Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a - // stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299). - Withheld []string `json:"withheld,omitempty"` - // Unchanged are the modules a merge it answers changed files in the directory of, whose recorded build source - // holds none of them, each with why (novox/hq issue 611, ADR 0267 §5): not built, because the build the mesh - // holds is their build. A delivery plan that read them whole named them; this says they matched what runs, - // rather than that the walk left them unbuilt. Modules keeps meaning what the walk builds. - Unchanged map[string]string `json:"unchanged,omitempty"` - // Batch is the window of a batch; nil once it is cut into a walk. - Batch *PlanBatch `json:"batch,omitempty"` - // Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one: - // built at that commit, not at the branch. - Alone bool `json:"alone,omitempty"` -} - -// PlanLeftOut is a merge a stop left out of the batch that walks a stopped walk's merges again (novox/hq ADR -// 0299): a merge stopped, or a later merge of its repository's branch, which contains the stopped one and so -// cannot be walked without delivering it. -type PlanLeftOut struct { - Repository string `json:"repository"` - Commit string `json:"commit"` - // Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge - // stopped itself. - Contains string `json:"contains,omitempty"` - // Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped - // merge, they would deliver it. - Shares []string `json:"shares,omitempty"` -} - -// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached. -type PlanKept struct { - Module string `json:"module"` - Machines []string `json:"machines"` -} - -// Waiting is whether the walk waits for its delivery's word. -func (p Plan) Waiting() bool { - return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil -} - -// PlanSaved is told every plan this process kept, after it is kept, with the inventory that kept it (novox/hq -// ADR 0239): the serving controller says it on the bus as `plan-moved`, and so does a command (the controller's -// run installs it for every verb, novox/hq ADR 0308 decision 1), so the save that starts or ends a walk is said -// whichever process made it. Nil says nothing; whoever follows a walk also asks for it, and finds an unsaid save -// by comparison. -var PlanSaved func(*Inventory, Plan) - -// ErrPlanMoved is a save against a plan written by somebody else since it was read. -var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read") - -// PlanModule is one module's state within a plan. -type PlanModule struct { - // State: asked, built, failed; empty for a module whose tier has not been asked yet. - State string `json:"state,omitempty"` - AskedAt *time.Time `json:"asked_at,omitempty"` - BuiltAt *time.Time `json:"built_at,omitempty"` - // BusWaitFrom is when the bus first did not answer an ask of this module (issue 1334): the module stays unasked and is - // asked again at the next look, until the bus has been silent for the bound, when the plan fails as for any refusal. - BusWaitFrom *time.Time `json:"bus_wait_from,omitempty"` - // SentAt is when the plan sent the machines running this module its new build, because a - // later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones - // after this. - SentAt *time.Time `json:"sent_at,omitempty"` - // First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue - // 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before - // the rest, and the rest are sent once that one reports it applied. Kept so a controller - // replaced while the plan waits on that report resumes the wait rather than sending again. The - // machine holding the bus is among them when its user list had to go first. - First []string `json:"first,omitempty"` - FirstAt *time.Time `json:"first_at,omitempty"` - Commit string `json:"commit,omitempty"` - Why string `json:"why,omitempty"` - // Placed is the edge that put the module after another, and the rule that drew it (novox/hq ADR 0419 clause 5): - // "after (verb before caller: .)". Empty for a module placed first by nothing. - Placed string `json:"placed,omitempty"` - // Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from ", naming the pull - // request's head, the trunk commit and the digests, or "built: ". Said in `plan-moved` and kept here. - Made string `json:"made,omitempty"` - // Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan - // matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt - // its module's name included. Empty in a plan from before it was kept, which is matched by - // module, or by repository and path, as before. - Build string `json:"build,omitempty"` - // Previous is the build the first machine ran of this module before the plan sent it the new one — - // the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back - // (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent - // is not known. - Previous string `json:"previous,omitempty"` - // Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a - // controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record. - Gate *PlanGate `json:"gate,omitempty"` - // GatedBy names the module of the same tier whose gate judges this one on its first machine: they - // went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for - // the module the gate is kept on, and for a plan from before tiers were sent whole. - GatedBy string `json:"gated_by,omitempty"` - // Rest is, per machine of the rest, the declaration the send after the first-node gate carried there (novox/hq ADR - // 0282 decision 6): the machine's first report of it, applied, is when the build runs there. - Rest map[string]SentDeclaration `json:"rest,omitempty"` - // Recorded is, for a module whose policy records, per machine running it with a move, which way the walk took it - // (novox/hq ADR 0377 §6): "went out: …" when the move recreates none of its containers and moves none of its - // data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it - // could not be judged. - Recorded map[string]string `json:"recorded,omitempty"` - // Key says the walk takes this module from the builder's done key and asked no build (novox/hq ADR 0501, ADR 0453 - // §5; issue 1851): from R4e every module but the builder's own. Its outcome is the key's, never a build-agent outcome. - Key bool `json:"key,omitempty"` - // KeyAt is the commit of its repository whose indexer's trunk value names the fingerprint taken (ADR 0501 §2); empty - // for a module of a repository the walk carries no commit of (a dependent through a base), taken at the value now. - KeyAt string `json:"key_at,omitempty"` - // Fingerprint is the fingerprint of the new form (`src2:`) the walk takes the module at, read from that trunk value; - // empty until the indexer has written it. - Fingerprint string `json:"fingerprint,omitempty"` -} - -// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine, -// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back. -type PlanGate struct { - // Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty - // for any other module, judged by its own health. - Component string `json:"component,omitempty"` - Machines []string `json:"machines"` - From string `json:"from,omitempty"` - To string `json:"to,omitempty"` - // Since is when the judging began: the first machine reported the new build applied. - Since *time.Time `json:"since,omitempty"` - // Sent is, per machine, the declaration the gate's send carried there (novox/hq issue 352): what a - // machine's report is held against. Absent on a gate kept before it was, which reads the report - // against the send made last, as before. - Sent map[string]SentDeclaration `json:"sent,omitempty"` - // Applied is, per machine, the time of the first report the gate read of its send applied: the - // node-engine started the build before it, so a witness window counted from it ends no earlier than the - // node-engine's own (novox/hq ADR 0351 decisions 1 and 4). - Applied map[string]time.Time `json:"applied,omitempty"` - // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that - // does not resets them. - Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two - // minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR - // 0282 decision 4 and ADR 0304 decision 3). Zero in a gate - // judged before it was said: three. - Wants int `json:"wants,omitempty"` - LastPass *time.Time `json:"last_pass,omitempty"` - // Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading - // rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352). - Faulted bool `json:"faulted,omitempty"` - // HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor - // counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3). - HealthySeen bool `json:"healthy_seen,omitempty"` - // Last is what the newest judging found wanting, while it still may pass. - Last string `json:"last,omitempty"` - // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since. - Verdict string `json:"verdict,omitempty"` - Why string `json:"why,omitempty"` - JudgedAt *time.Time `json:"judged_at,omitempty"` - Took string `json:"took,omitempty"` - // Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why. - Rollback string `json:"rollback,omitempty"` - // Kept says a passing verdict was written to the gate's records. - Kept bool `json:"kept,omitempty"` - // Carried is every module whose build moved on the judged machines with the send — the plan's own - // module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a - // pass is its verdict too, and one that fails is put back. - Carried []CarriedMove `json:"carried,omitempty"` - // Failing names the modules the last judging found wanting. - Failing []string `json:"failing,omitempty"` - // Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged, - // or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's. - Healthy map[string]int `json:"healthy,omitempty"` - // Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never - // a reason to fail. - Waits map[string]string `json:"waits,omitempty"` - // Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its - // pass, and is not put back. - Passing []string `json:"passing,omitempty"` - // HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery - // after the one before (issue 318 review). - HealthyAt map[string]time.Time `json:"healthy_at,omitempty"` - // Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the - // first reason: the send fails, and the modules beside them are judged to their own verdict first. - Broken []string `json:"broken,omitempty"` - BrokenWhy string `json:"broken_why,omitempty"` - // Returned names the broken modules already put back, at once, while the rest of the send is judged. - Returned []string `json:"returned,omitempty"` - // Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that - // counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept. - Readings []GateReading `json:"readings,omitempty"` - // Grants is the grants step taken before this gate's send (novox/hq issue 490): the bus's user list sent - // alone to the machine holding the bus, every build there kept, so what the send newly grants is on the - // bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus. - Grants *GrantsStep `json:"grants,omitempty"` - // Busy is the wait for a busy container on a judged machine (novox/hq ADR 0335 §4): while the machine - // reports its module's container left running because it said busy, the judging waits and has not - // started; once it reports the container replaced, Since is moved to then. Nil when nothing waited. - Busy *GateBusy `json:"busy,omitempty"` - // Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that - // takes over reads the machine's last send before making it, and never sends it twice. - Pending bool `json:"pending,omitempty"` - // Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left, - // neither failed nor put back. - Silent bool `json:"silent,omitempty"` - // TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of - // the same build on the same machine, which this gate goes on from instead of starting from zero. - TookOver string `json:"took_over,omitempty"` -} - -// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR -// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found. -// A module's own count is taken only for the modules this gate judges. -func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) { - if from == nil { - return - } - if from.Since != nil { - since := *from.Since - g.Since = &since - } - for n, s := range from.Sent { - if slices.Contains(g.Machines, n) { - if g.Sent == nil { - g.Sent = map[string]SentDeclaration{} - } - g.Sent[n] = s - } - } - for n, at := range from.Applied { - if slices.Contains(g.Machines, n) { - if g.Applied == nil { - g.Applied = map[string]time.Time{} - } - g.Applied[n] = at - } - } - g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last - if from.LastPass != nil { - at := *from.LastPass - g.LastPass = &at - } - for _, m := range modules { - if n, counted := from.Healthy[m]; counted { - if g.Healthy == nil { - g.Healthy = map[string]int{} - } - g.Healthy[m] = n - } - if at, counted := from.HealthyAt[m]; counted { - if g.HealthyAt == nil { - g.HealthyAt = map[string]time.Time{} - } - g.HealthyAt[m] = at - } - } - g.Readings = append([]GateReading(nil), from.Readings...) - g.TookOver = id -} - -// GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4). -type GateBusy struct { - Machine string `json:"machine"` - Module string `json:"module"` - Container string `json:"container"` - // Since is when the machine's first apply that would have replaced it ran, Until when its node-engine - // replaces it at the latest; Ended when the machine reported it replaced and the judging began — or when - // the controller stopped waiting past the engine's bound, which PastBound says. - Since time.Time `json:"since"` - Until time.Time `json:"until"` - // Seen is when this controller first read the wait, on its own clock: the gate's bound runs from it, never - // from the engine's word alone. - Seen time.Time `json:"seen,omitzero"` - Ended *time.Time `json:"ended,omitempty"` - PastBound bool `json:"past_bound,omitempty"` - // Said is the last output of its busy command, as the machine reported it. - Said string `json:"said,omitempty"` -} - -// Waiting says the gate waits for a busy container and has not started. -func (g *PlanGate) Waiting() bool { return g != nil && g.Busy != nil && g.Busy.Ended == nil } - -// Words is the wait in a line, as the walk says it. -func (b *GateBusy) Words() string { - said := "" - if b.Said != "" { - said = "; its busy command says: " + b.Said - } - return fmt.Sprintf("waits for a busy container on %s: %s holds work, and is replaced once it says idle, by %s "+ - "at the latest (novox/hq ADR 0335)%s", b.Machine, b.Container, b.Until.UTC().Format(time.RFC3339), said) -} - -// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq -// issue 490): to which machine, when, and, when it could not be sent, why — the send went on without it. -type GrantsStep struct { - Node string `json:"node"` - At *time.Time `json:"at,omitempty"` - Failed string `json:"failed,omitempty"` -} - -// GateReading is one reading of a first-node gate. -type GateReading struct { - At time.Time `json:"at"` - Healthy bool `json:"healthy"` - // Said is what a reading that did not pass found wanting. - Said string `json:"said,omitempty"` -} - -// maxReadings bounds a first-node gate's readings: a judging that never passes reads every few seconds for ten minutes. -const maxReadings = 24 - -// Read keeps one reading: a pass always, and a reading that did not pass only when the one before passed or -// there is none, so a judging waiting for a module to start keeps one line of it, not hundreds. -func (g *PlanGate) Read(at time.Time, healthy bool, said string) { - if !healthy && len(g.Readings) > 0 && !g.Readings[len(g.Readings)-1].Healthy { - return - } - if r := []rune(said); len(r) > 200 { - said = string(r[:200]) - } - g.Readings = append(g.Readings, GateReading{At: at, Healthy: healthy, Said: said}) - if len(g.Readings) > maxReadings { - g.Readings = g.Readings[len(g.Readings)-maxReadings:] - } -} - -// CarriedMove is one module's build moving on a machine with a gated send. -type CarriedMove struct { - Module string `json:"module"` - Node string `json:"node"` - From string `json:"from,omitempty"` - To string `json:"to"` - Build string `json:"build,omitempty"` - // Recreates is what the send does to the module's containers, in the mesh's words — how many it - // recreates, and whether with a new image or only their declaration (novox/hq ADR 0245); empty when - // it recreates none, or when the build the machine ran is not known. - Recreates string `json:"recreates,omitempty"` -} - -// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build -// that waits for a gate, sent one machine at a time, each judged before the next. -type PlanRelease struct { - Order []string `json:"order"` - Next int `json:"next"` - // Gate is the first machine being judged; nil between machines. - Gate *PlanGate `json:"gate,omitempty"` - // Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by - // its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2). - Rest bool `json:"rest,omitempty"` - Gates map[string]*PlanGate `json:"gates,omitempty"` - // Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send - // within ApplySilentAfter (ADR 0353 §3 and §4). - Failed []string `json:"failed,omitempty"` - Silent []string `json:"silent,omitempty"` - Done []string `json:"done,omitempty"` - // Skipped are the machines not heard from when their turn came, left as they were. - Skipped []string `json:"skipped,omitempty"` - // By is the person who released it, empty when the mesh did. - By string `json:"by,omitempty"` - // Settings is set on a settings walk (novox/hq ADR 0344): not the builds waiting for a gate, but a settings - // layer the operator approved, sent to the machines it reaches one at a time, its module judged on each. - Settings *PlanSettings `json:"settings,omitempty"` -} - -// PlanSettings is what a settings walk sends (novox/hq ADR 0344): the module whose layer the operator approved, the -// layer's scope, the ask that approved it and what changed, in words. -type PlanSettings struct { - Module string `json:"module"` - // Node is the machine the layer is for, empty for the whole mesh. - Node string `json:"node,omitempty"` - // Ask is the proposal the operator approved, and Change what its layer changed ("~ prefs"). - Ask string `json:"ask"` - Change string `json:"change,omitempty"` -} - -// The states a plan passes through. -const ( - PlanBuilding = "building" - PlanRolling = "rolling" - PlanDone = "done" - PlanFailed = "failed" - // PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq - // issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it. - PlanSuperseded = "superseded" - // PlanAssembling is a batch whose merge window is open (novox/hq ADR 0276), and PlanQueued one whose - // window closed while a walk is open: neither is a walk yet. Cut, a batch keeps its id and is building. - PlanAssembling = "assembling" - PlanQueued = "queued" -) - -// Open says whether the plan is still being worked. -func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling } - -// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing. -// -// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if -// the plan is still at the revision it was read at — a new one only if it does not exist — and refused -// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records. -// On success p's revision and epoch are the ones written, so the caller may save it again. -func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { - return i.SavePlansMoving(ctx, []*Plan{p}, nil) -} - -// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276). -type MergeMove struct { - Repository, Commit, Plan string - Alone bool - // Event replaces the merge's kept announcement when it is put back to waiting (Plan empty); nil leaves it. - Event json.RawMessage -} - -// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one -// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never -// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by -// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once -// the transaction is committed, in order. -func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error { - c, err := i.theChain(ctx) - if err != nil { - return err - } - return i.chainSavePlans(ctx, c, plans, moves) -} - -// short is a commit as a person reads it. -func short(commit string) string { - if len(commit) > 8 { - return commit[:8] - } - return commit -} - -// OpenPlans is every plan still being worked, oldest first. -func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) -} - -// PlansSince is every plan made after a moment, and every plan still being worked, oldest first. -func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated) -} - -// RecentPlans is the last few plans, newest first, open or not — what the overview shows. -func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest) - if err == nil && limit >= 0 && len(plans) > limit { - plans = plans[:limit] - } - return plans, err -} - -// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh -// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it -// recorded when its merge was made. -func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return Plan{}, false, err - } - return i.chainNewestMergeOf(ctx, c, repository, branch) -} - -// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a -// caller of the plans verb. -func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return Plan{}, err - } - return i.chainPlanByID(ctx, c, id) -} diff --git a/internal/inventory/plans_test.go b/internal/inventory/plans_test.go deleted file mode 100644 index 819b393c..00000000 --- a/internal/inventory/plans_test.go +++ /dev/null @@ -1,225 +0,0 @@ -//go:build integration - -package inventory - -import ( - "reflect" - "testing" - "time" -) - -// A plan is a record the mesh keeps and resumes (novox/hq ADR 0162): written whole, read back open, -// advanced, and gone from the open ones when done. -func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(), - State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}, {"shop"}}, - Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}, "shop": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - open, err := inv.OpenPlans(ctx) - if err != nil || len(open) != 1 || open[0].ID != "plan-1" || len(open[0].Tiers) != 3 { - t.Fatalf("the plan was not kept whole: %v %+v", err, open) - } - // Another controller picks it up where it was left: a tier advanced and a module built. - now := time.Now().UTC() - resumed := open[0] - resumed.Tier = 1 - resumed.Modules["mesh-tools"].State = "built" - resumed.Modules["mesh-tools"].BuiltAt = &now - resumed.State = PlanRolling - resumed.Note = "tier 0 built; waiting for builder on anchor to be applied" - if err := inv.SavePlan(ctx, &resumed); err != nil { - t.Fatal(err) - } - again, err := inv.PlanByID(ctx, "plan-1") - if err != nil || again.Tier != 1 || again.Modules["mesh-tools"].State != "built" || again.State != PlanRolling { - t.Fatalf("the advanced plan did not come back as left: %v %+v", err, again) - } - again.State = PlanDone - if err := inv.SavePlan(ctx, &again); err != nil { - t.Fatal(err) - } - if open, _ = inv.OpenPlans(ctx); len(open) != 0 { - t.Fatalf("a done plan is not open: %+v", open) - } - if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanDone { - t.Fatalf("a done plan is still among the recent ones: %+v", recent) - } -} - -// novox/hq issue 254: a plan keeps the branch its merge went into, and a superseded plan is not open. -func TestASupersededPlanIsNotOpen(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "abc", - Created: time.Now().UTC(), State: PlanBuilding, Tiers: [][]string{{"gitea"}}, - Modules: map[string]*PlanModule{"gitea": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - kept, err := inv.PlanByID(ctx, "plan-1") - if err != nil || kept.Branch != "main" { - t.Fatalf("the branch was not kept: %v %+v", err, kept) - } - kept.State = PlanSuperseded - kept.Note = "superseded at tier 0 by plan-2" - if err := inv.SavePlan(ctx, &kept); err != nil { - t.Fatal(err) - } - if open, err := inv.OpenPlans(ctx); err != nil || len(open) != 0 { - t.Fatalf("a superseded plan is still open: %v %+v", err, open) - } - if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanSuperseded { - t.Fatalf("a superseded plan is not among the recent ones as superseded: %+v", recent) - } -} - -// novox/hq issue 349 (review of the follow-up): the newest merge of a branch is the one merged last, whatever -// order the plans were made in, in any state; a tie is broken by the plan made last; a release, another -// branch and another repository are never it; and its time is kept to the nanosecond. -func TestTheNewestMergeOfABranchIsTheOneMergedLast(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - t0 := time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC) - save := func(id, repo, branch string, merged, created time.Time, state string, release bool) { - t.Helper() - p := Plan{ID: id, Repository: repo, Branch: branch, Commit: id + "-commit", Merged: merged, Created: created, - State: state, Tiers: [][]string{}, Modules: map[string]*PlanModule{}} - if release { - p.Release = &PlanRelease{} - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - } - if _, found, err := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); err != nil || found { - t.Fatalf("a branch with no plan: %v %v", found, err) - } - // Made in the other order than merged: the later merge's plan made first, and done. - save("plan-later", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0, PlanDone, false) - save("plan-earlier", "novox/mesh-catalog", "main", t0.Add(time.Minute), t0.Add(5*time.Minute), PlanRolling, false) - save("plan-other-branch", "novox/mesh-catalog", "release", t0.Add(time.Hour), t0, PlanRolling, false) - save("plan-other-repo", "novox/mesh-controller", "main", t0.Add(time.Hour), t0, PlanRolling, false) - save("release-1", "novox/mesh-catalog", "main", t0.Add(time.Hour), t0, PlanRolling, true) - save("plan-unknown", "novox/mesh-catalog", "main", time.Time{}, t0.Add(time.Hour), PlanRolling, false) - p, found, err := inv.NewestMergeOf(ctx, "Novox/Mesh-Catalog", "main") - if err != nil || !found || p.ID != "plan-later" { - t.Fatalf("the newest merge: %s %v %v", p.ID, found, err) - } - if !p.Merged.Equal(t0.Add(2*time.Minute + 250*time.Millisecond)) { - t.Fatalf("its merge time was not kept to the nanosecond: %s", p.Merged) - } - // The same merge time: the plan made last. - save("plan-again", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0.Add(time.Minute), PlanBuilding, false) - if p, _, _ := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); p.ID != "plan-again" { - t.Fatalf("one merge time, two plans: %s", p.ID) - } -} - -// novox/hq issue 352: what a machine was last sent is read back by name with its sequence, a report keeps -// the declaration it was about and that declaration's sequence, and a gate's sends are kept with the plan. -func TestASendAndAReportAreKnownByTheirDeclaration(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if _, found, err := inv.SentTo(ctx, "anchor"); err != nil || found { - t.Fatalf("a machine never sent anything: %v %v", found, err) - } - if _, _, err := inv.SentTo(ctx, "nobody"); err == nil { - t.Fatal("a machine that does not exist was answered") - } - seq, err := inv.NextSequence(ctx, record.ID) - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d-1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - sent, found, err := inv.SentTo(ctx, "anchor") - if err != nil || !found || sent.Digest != "d-1" || sent.Sequence != seq { - t.Fatalf("sent %+v %v %v", sent, found, err) - } - if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Node: "anchor", Outcome: OutcomeApplied, Declared: "d-1", Applied: 1}, - ReportOrder{Sequence: seq, ReportSequence: 1}, func(ReportOrder) bool { return false }); err != nil { - t.Fatal(err) - } - reports, err := inv.LastReports(ctx) - if err != nil || len(reports) != 1 || reports[0].Declared != "d-1" || reports[0].ReportedSequence != seq || !reports[0].Current { - t.Fatalf("reports %+v %v", reports, err) - } - // Sent again, unreported: the report is no longer on the last send, and is still on the first. - if err := inv.RecordSent(ctx, record.ID, "d-2", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - reports, _ = inv.LastReports(ctx) - if reports[0].Current || !sent.ReportsOn(reports[0]) { - t.Fatalf("after a newer send: %+v", reports[0]) - } - at := time.Now().UTC() - p := Plan{ID: "plan-352", Repository: "novox/x", Commit: "c", Created: at, State: PlanRolling, Tiers: [][]string{{"app"}}, - Modules: map[string]*PlanModule{"app": {Gate: &PlanGate{Machines: []string{"anchor"}, Since: &at, - Sent: map[string]SentDeclaration{"anchor": sent}}}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - kept, err := inv.PlanByID(ctx, "plan-352") - if err != nil || !reflect.DeepEqual(kept.Modules["app"].Gate.Sent["anchor"], sent) { - t.Fatalf("the gate's send was not kept with the plan: %+v %v", kept.Modules["app"].Gate, err) - } -} - -// A controller build's reach of the store's schema is kept by its version, and the store's own is read. -func TestASchemaReachIsKeptByBuild(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - applied, err := inv.SchemaApplied(ctx) - if err != nil || applied < 87 { - t.Fatalf("applied %d %v", applied, err) - } - if _, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || known { - t.Fatalf("an unrecorded build: %v %v", known, err) - } - if err := inv.RecordSchemaReach(ctx, "", 87); err == nil { - t.Fatal("a reach without a build was recorded") - } - if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 86); err != nil { - t.Fatal(err) - } - if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 87); err != nil { - t.Fatal(err) - } - if reach, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || !known || reach != 87 { - t.Fatalf("reach %d %v %v", reach, known, err) - } -} - -// A plan keeps its backups through the store, timestamps to the nanosecond (novox/hq ADR 0404, issue 1065). -func TestAPlanKeepsItsBackupsInTheStore(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - at := time.Now().UTC() - p := Plan{ID: "plan-backups", Repository: "novox/mesh-catalog", Commit: "abc", Created: at, State: PlanBuilding, - Tiers: [][]string{{"mail"}}, Modules: map[string]*PlanModule{"mail": {}}, - Backups: &PlanBackups{State: BackupPending, NoHolder: []string{"bare"}, - Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupPending, AskedAt: &at}}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - got, err := inv.PlanByID(ctx, p.ID) - if err != nil || got.Backups == nil || !reflect.DeepEqual(got.Backups, p.Backups) { - t.Fatalf("kept %+v (%v), wrote %+v", got.Backups, err, p.Backups) - } - got.Backups.State = BackupDone - if err := inv.SavePlan(ctx, &got); err != nil { - t.Fatal(err) - } - if again, _ := inv.PlanByID(ctx, p.ID); again.Backups == nil || again.Backups.State != BackupDone { - t.Fatalf("the change was not kept: %+v", again.Backups) - } -} diff --git a/internal/inventory/provisioned.go b/internal/inventory/provisioned.go index ac2779bd..ba2c8f0e 100644 --- a/internal/inventory/provisioned.go +++ b/internal/inventory/provisioned.go @@ -1,8 +1,6 @@ package inventory import ( - "fmt" - "sort" "time" ) @@ -37,84 +35,3 @@ const ( ProvFailed = "failed" ProvRemoved = "removed" ) - -// ProvLookup answers a client's newest report of a module on a machine; false when it has said nothing of it. -type ProvLookup func(node, module string) (ProvReport, bool) - -// OutcomeProvisioningFailed begins the outcome of a send whose module's files the provisioner failed to place. -const OutcomeProvisioningFailed = "provisioning failed" - -// ProvisionedSend is a machine's report of a send, with its provisioner client's reports of every module the send -// handed to it: what a walk counts as that machine's report. engine and reported are the machine's own report and -// whether it said one. A failed client report is the send's outcome, whatever the machine said; none, or one not -// yet applied on the build sent, leaves the send unreported. -func ProvisionedSend(node string, send SentDeclaration, engine AppliedReport, reported bool, prov ProvLookup) (AppliedReport, bool) { - if len(send.Provisioned) == 0 { - return engine, reported - } - // The machine's own failure, or its wait for a busy container, is its report whatever the client says. - if reported && (engine.Outcome != OutcomeApplied || engine.Waiting) { - return engine, true - } - modules := make([]string, 0, len(send.Provisioned)) - for m := range send.Provisioned { - modules = append(modules, m) - } - sort.Strings(modules) - at, placed := engine.At, true - for _, m := range modules { - r, said := ProvReport{}, false - if prov != nil { - r, said = prov(node, m) - } - switch { - case said && r.Status == ProvFailed && aboutTheSend(r, send, send.Provisioned[m]): - return AppliedReport{At: r.At, Outcome: fmt.Sprintf("%s (%s: %s)", OutcomeProvisioningFailed, m, - orNoReason(r.Reason))}, true - case said && r.Status == ProvApplied && r.Templates == send.Provisioned[m]: - if r.At.After(at) { - at = r.At - } - default: - placed = false - } - } - if !reported || !placed { - return AppliedReport{}, false - } - engine.At = at - return engine, true -} - -// Only is the send narrowed to what it handed to the provisioner for the named modules: a delivery waits on the -// modules it carries, never on another module the same machine's declaration hands (ADR 0362 §1-2, issue 840). -// A send handing nothing is returned as it is. -func (s SentDeclaration) Only(modules ...string) SentDeclaration { - if len(s.Provisioned) == 0 { - return s - } - kept := map[string]string{} - for _, m := range modules { - if d, ok := s.Provisioned[m]; ok { - kept[m] = d - } - } - s.Provisioned = kept - return s -} - -// aboutTheSend says a client's failure is about what the send handed: on the build's templates, or, said before it -// had any to name (its templates could not be fetched), no earlier than the send. -func aboutTheSend(r ProvReport, send SentDeclaration, digest string) bool { - if r.Templates != "" { - return r.Templates == digest - } - return send.At.IsZero() || !r.At.Before(send.At) -} - -func orNoReason(s string) string { - if s == "" { - return "no reason given" - } - return s -} diff --git a/internal/inventory/provisioned_test.go b/internal/inventory/provisioned_test.go deleted file mode 100644 index 9057168d..00000000 --- a/internal/inventory/provisioned_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package inventory - -import ( - "strings" - "testing" - "time" -) - -// A walk counts a provisioned module applied on a machine only once its files are placed (novox/hq ADR 0362, design -// 57 P6, issue 672). The walk of issue 582 (one module, three machines of the rest), with g14's send having handed -// dunst to the provisioner: g14's own report comes 5 s after its send, shanks's, the last of the others, 15 s -// after its own. - -const dunstTemplates = "sha256:" + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" - -func walkHandingDunst(t *testing.T) sendRest582 { - t.Helper() - c := walk582(t) - s := c.walk.Modules["build-agent"].Rest["g14"] - s.Provisioned = map[string]string{"build-agent": dunstTemplates} - c.walk.Modules["build-agent"].Rest["g14"] = s - return c -} - -// provisioning is a walk's lookup with the machines' clients saying reports. -func (w sendRest582) provisioning(reports map[string]ProvReport) AppliedLookup { - return func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - r, ok := w.lookup(node, send, walkSent) - return ProvisionedSend(node, send, r, ok, func(node, module string) (ProvReport, bool) { - p, said := reports[node+"."+module] - return p, said - }) - } -} - -func restOf(w *WalkPhases, node string) RestReport { - for _, r := range w.Rest { - if r.Node == node { - return r - } - } - return RestReport{} -} - -// The node-engine's report alone does not end the walk: it waits for prov.applied; with it, the walk ends, and -// its delivery time runs to the later of the two reports. -func TestAWalkWaitsForTheProvisionersReportAndEndsAtTheLater(t *testing.T) { - c := walkHandingDunst(t) - g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At - - w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(nil)) - if w.End != nil || restOf(w, "g14").State != RestWaiting || - !strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") { - t.Fatalf("the walk did not wait for g14's prov.applied: end %v, g14 %+v, apply %+v", w.End, restOf(w, "g14"), - phase(w, PhaseApply, -1)) - } - - // A report on an older build of the templates is not this send's. - older := map[string]ProvReport{"g14.build-agent": {Status: ProvApplied, Templates: "sha256:" + strings.Repeat("0", 64), - At: g14Sent.Add(2 * time.Second)}} - if w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(older)); w.End != nil { - t.Fatalf("a report on an older build ended the walk at %v", w.End) - } - - placed := g14Sent.Add(50 * time.Second) // after shanks's report, 35 s after the judgement - w = c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{ - "g14.build-agent": {Status: ProvApplied, Templates: dunstTemplates, At: placed}})) - sumsUp(t, w) - if w.End == nil || !w.End.Equal(placed) { - t.Fatalf("the walk ends at %v, want the provisioner's report at %s: %+v", w.End, placed, w.Rest) - } - if g := restOf(w, "g14"); g.State != RestApplied || !g.Reported.Equal(placed) { - t.Fatalf("g14 is not applied at its later report: %+v", g) - } -} - -// A failed prov.applied is the machine's failed apply, named in the walk's end with its reason. -func TestAFailedProvisioningIsAFailedApply(t *testing.T) { - c := walkHandingDunst(t) - g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At - w := c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{ - "g14.build-agent": {Status: ProvFailed, Reason: "the templates cannot be fetched", At: g14Sent.Add(3 * time.Second)}})) - if w.End != nil { - t.Fatalf("a failed provisioning ended the walk: %v", w.End) - } - if g := restOf(w, "g14"); g.State != RestFailed || !strings.Contains(g.Outcome, "the templates cannot be fetched") { - t.Fatalf("g14 is not named failed with the reason: %+v", g) - } - if said := phase(w, PhaseApply, -1).Said; !strings.Contains(said, "not applied on g14 (provisioning failed (build-agent: "+ - "the templates cannot be fetched))") { - t.Fatalf("the walk's end does not name g14's failure: %q", said) - } - // A failure said before the send, of no build, is not this send's. - stale := map[string]ProvReport{"g14.build-agent": {Status: ProvFailed, Reason: "old", At: g14Sent.Add(-time.Hour)}} - if g := restOf(c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(stale)), "g14"); g.State != RestWaiting { - t.Fatalf("a failure from before the send was read as this send's: %+v", g) - } -} - -// None within ApplySilentAfter names the machine silent, never counted applied; the others end the walk. -func TestNoProvisioningReportWithinTheBoundIsSilence(t *testing.T) { - c := walkHandingDunst(t) - g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At - waiting := map[string]ProvReport{"g14.build-agent": {Status: ProvWaiting, WaitsFor: []string{"config:font-size"}, - At: g14Sent.Add(3 * time.Second)}} - w := c.walk.WalkPhases(g14Sent.Add(ApplySilentAfter+time.Minute), c.provisioning(waiting)) - sumsUp(t, w) - if g := restOf(w, "g14"); g.State != RestSilent { - t.Fatalf("g14 is not silent past ApplySilentAfter: %+v", g) - } - if len(w.Silent) != 1 || w.Silent[0] != "g14" || w.End == nil || !w.End.Equal(c.judged.Add(35*time.Second)) { - t.Fatalf("the walk does not end on the others, naming g14 silent: end %v, silent %v", w.End, w.Silent) - } -} - -// A send that handed nothing ends on the node-engine's report alone, as before. -func TestASendThatHandedNothingEndsOnTheEnginesReport(t *testing.T) { - c := walk582(t) - before := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - now := c.walk.WalkPhases(c.judged.Add(time.Hour), c.provisioning(nil)) - if before.End == nil || now.End == nil || !now.End.Equal(*before.End) { - t.Fatalf("a walk handing nothing ends differently: before %v, now %v", before.End, now.End) - } -} diff --git a/internal/inventory/provisioning.go b/internal/inventory/provisioning.go index 09d58bfd..3fdeb641 100644 --- a/internal/inventory/provisioning.go +++ b/internal/inventory/provisioning.go @@ -408,6 +408,18 @@ func sortedPlanKeys(plans map[string]Provisioning) []string { // (migration 0100) by module, and carried by the assignments mirror in each handed module's assignment. So the // client is handed the plan of the build its machine was sent, and a controller started afresh mirrors the same. +// RecordComposedPlans keeps the provisioner's plan of each module a node's composition hands to it, as JSON by module +// (`: plan and required-by`), or none: what SentProvisioningPlans reads for the assignments mirror (design 57 P6). +// With no send left, the composition is where the plans are composed (novox/hq ADR 0539 clause 5). +func (i *Inventory) RecordComposedPlans(ctx context.Context, node string, plans []byte) error { + var kept any + if len(plans) > 0 { + kept = plans + } + _, err := i.store.Pool().Exec(ctx, `update node set sent_provisioning = $2 where name = $1`, node, kept) + return err +} + // SentProvisioningPlans is every plan the machines were last sent, keyed `.`: what // SetProvisioningPlans is given. A plan of a module only pulled in by another is kept with what required it, and is // mirrored in a value of its own until a send drops it (issue 765). A plan of a module assigned when it was sent and diff --git a/internal/inventory/registered_held_integration_test.go b/internal/inventory/registered_held_integration_test.go deleted file mode 100644 index e690c21f..00000000 --- a/internal/inventory/registered_held_integration_test.go +++ /dev/null @@ -1,59 +0,0 @@ -//go:build integration - -package inventory - -import ( - "encoding/json" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - manifestOf := func(image string) catalogue.Manifest { - return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": image}}} - } - record := func(id, image string, asked time.Time) Build { - t.Helper() - raw, _ := json.Marshal(manifestOf(image)) - b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw, - Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111" - y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222" - before := record("x", x, time.Now().Add(-2*time.Hour)) - failed := record("y", y, time.Now().Add(-time.Minute)) - if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit, - Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - running, err := inv.RegisteredHeld(ctx) - if err != nil { - t.Fatal(err) - } - if running["one/one"] == y { - t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"]) - } - if got := running["one/one"]; got != "" && got != x { - t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x) - } -} diff --git a/internal/inventory/schema.go b/internal/inventory/schema.go index fa971828..40aaee29 100644 --- a/internal/inventory/schema.go +++ b/internal/inventory/schema.go @@ -3,8 +3,6 @@ package inventory import ( "context" "errors" - - "github.com/jackc/pgx/v5" ) // What a controller build knows of the store's schema (novox/hq issue 352): the highest migration it @@ -23,17 +21,6 @@ func (i *Inventory) RecordSchemaReach(ctx context.Context, build string, reach i return err } -// SchemaReachOf is the highest migration a build carries, as it recorded when it served; false for a -// build that never did. -func (i *Inventory) SchemaReachOf(ctx context.Context, build string) (int, bool, error) { - var reach int - err := i.store.Pool().QueryRow(ctx, `select reach from controller_schema where build = $1`, build).Scan(&reach) - if errors.Is(err, pgx.ErrNoRows) { - return 0, false, nil - } - return reach, err == nil, err -} - // SchemaApplied is the highest migration the store has applied. func (i *Inventory) SchemaApplied(ctx context.Context) (int, error) { var n *int diff --git a/internal/inventory/seats_test.go b/internal/inventory/seats_test.go index fb6cb9c7..4410e9f6 100644 --- a/internal/inventory/seats_test.go +++ b/internal/inventory/seats_test.go @@ -126,7 +126,7 @@ func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) { if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { t.Fatal(err) } - old := `[{"name":"push","description":"an older push","input":{"type":"object","properties":{"node":{"type":"string"}}}}, + old := `[{"name":"assign","description":"an older assign","input":{"type":"object","properties":{"node":{"type":"string"}}}}, {"name":"newer","description":"a verb of a newer build"}]` if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`, []byte(old), catalogue.ControllerSeatName); err != nil { @@ -147,9 +147,10 @@ func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) { } } } - props, _ := verbs["push"].Input["properties"].(map[string]any) - if _, takesWhy := props["why"]; !takesWhy || verbs["push"].Description == "an older push" { - t.Fatalf("push in the row is still the older build's: %+v", verbs["push"]) + // assign, a verb release 6 keeps (push went with the plans, ADR 0539): the row takes the binary's arguments. + props, _ := verbs["assign"].Input["properties"].(map[string]any) + if _, takesModule := props["module"]; !takesModule || verbs["assign"].Description == "an older assign" { + t.Fatalf("assign in the row is still the older build's: %+v", verbs["assign"]) } if _, kept := verbs["newer"]; !kept { t.Fatal("a verb only the row has was dropped") diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 7af5ea93..1df09e40 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "runtime" "slices" "sort" "strings" @@ -21,6 +22,31 @@ import ( // The table holds nothing usable — see the migration and internal/secrets for why that is the // design rather than an inconvenience. +// SecretTrace, when set, is told of every write of a pair credential's row — a made or accepted one, a rotated one discarded — +// with the callers that led to it. A test sets it to see who wrote what; the controller never does. +var SecretTrace func(op, what string) + +func traceSecret(op, name, consumer, module, provider string) { + if SecretTrace == nil { + return + } + pcs := make([]uintptr, 8) + n := runtime.Callers(3, pcs) + frames := runtime.CallersFrames(pcs[:n]) + var callers []string + for { + f, more := frames.Next() + if i := strings.LastIndex(f.Function, "/"); i >= 0 { + f.Function = f.Function[i+1:] + } + callers = append(callers, f.Function) + if !more || len(callers) >= 5 { + break + } + } + SecretTrace(op, fmt.Sprintf("%s for %s/%s from %s, by %s", name, consumer, module, provider, strings.Join(callers, " < "))) +} + // Secret is one provision's credential, sealed to each end. type Secret struct { Name string @@ -146,6 +172,7 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul if err != nil { return Secret{}, err } + traceSecret("made", name, consumer, consumerModule, provider) return Secret{Name: name, Consumer: consumer, ConsumerModule: consumerModule, Local: local, Provider: provider, ForConsumer: made.ForConsumer, ForProvider: made.ForProvider, @@ -276,6 +303,9 @@ func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, consumerMo `delete from secret where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4 and local = $5`, name, consumerNode.ID, consumerModule, providerNode.ID, local) + if err == nil { + traceSecret("discarded", name, consumer, consumerModule, provider) + } return err } diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go deleted file mode 100644 index 6ed98725..00000000 --- a/internal/inventory/sends.go +++ /dev/null @@ -1,364 +0,0 @@ -package inventory - -import ( - "context" - "fmt" - "time" - - "github.com/jackc/pgx/v5" -) - -// The assignment generation and the record of every send (novox/hq issue 234, migration 0094). -// -// A declaration carries the generation of the assignments it was composed from, and a machine refuses one -// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later -// send carrying an older view. And every send is written down with who sent it, from which generation and -// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at. - -// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes -// on a busy machine; the record is for finding who sent what, not an archive. -const sendsKept = 200 - -// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same -// transaction as every change to what is assigned where (a trigger on the assignment table). -func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) { - var g int64 - if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil { - return 0, fmt.Errorf("reading the assignment generation: %w", err) - } - return g, nil -} - -// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more -// than that, or what it already was when it is past it. It never lowers it. -// -// For one case only: a machine refused a send composed from the generation the mesh holds now, so the -// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after -// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed -// from the assignments the store holds now. A stale sender never meets this: its generation is below the -// counter, and the counter is not touched for it. -func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) { - var g int64 - err := i.store.Pool().QueryRow(ctx, - `update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g) - if err != nil { - return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err) - } - return g, nil -} - -// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the -// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed. -func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) { - var g *int64 - if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil { - return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err) - } - if g == nil { - return 0, nil - } - return *g, nil -} - -// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id. -func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) { - var reads bool - if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil { - return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err) - } - return reads, nil -} - -// RecordReadsGeneration keeps what a machine's latest report said of reading a generation. -func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error { - _, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads) - return err -} - -// ReadsModules says a machine's node-engine said it reads a module per resource and a build per module (issue 1145). -func (i *Inventory) ReadsModules(ctx context.Context, id string) (bool, error) { - var reads bool - if err := i.store.Pool().QueryRow(ctx, `select reads_modules from node where id = $1`, id).Scan(&reads); err != nil { - return false, fmt.Errorf("reading whether %s reads modules: %w", id, err) - } - return reads, nil -} - -// RecordReadsModules keeps what a machine's latest report said of reading modules. -func (i *Inventory) RecordReadsModules(ctx context.Context, id string, reads bool) error { - _, err := i.store.Pool().Exec(ctx, `update node set reads_modules = $2 where id = $1`, id, reads) - return err -} - -// Send is one declaration sent to a machine, as the mesh records it. -type Send struct { - // Node is the machine's id; NodeName its name, filled where it is read back. - Node string - NodeName string - Sequence int64 - // Epoch is the lease epoch its sender acted under, zero for one that acted under none. - Epoch int64 - // Sender is who sent it, in words: the caller, and the process and build that composed it. - Sender string - // Generation is the assignment generation it was composed from, zero when not known. - Generation int64 - Digest string - // Modules are the modules it named. - Modules []string - // Provisioning is the provisioner's plan of each module the send handed to it, by module, as JSON (novox/hq - // ADR 0349 §4, design 57 P6): kept on the machine's record with the send, nil when it handed none. Never in - // the record of the send itself. - Provisioning []byte - SentAt time.Time - // RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the - // generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to - // because the refusal showed it behind the machine, zero when it was not. - RefusedAt *time.Time - RefusedApplied int64 - CounterRaisedTo int64 - // Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown. - Recorded bool - // Answer is what the machine last reported of this send; filled only where a last send is read back - // (LastSend, LastSends), the zero answer elsewhere. - Answer SendAnswer -} - -// SendAnswer is what a machine reported of one send, read from its last report when that report is about the -// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why. -// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it. -type SendAnswer struct { - // Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it. - Outcome string - // Refused is the machine's own words when it refused the declaration whole. - Refused string - // Failed is how many of its resources failed, when some did. - Failed int - // At is when the machine reported it; nil when it has not. - At *time.Time -} - -// unknownSender is the sender of a refused sequence the mesh has no record of sending. -const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " + - "hand, before sends were recorded, or by a controller whose record was not written)" - -// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it -// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the -// caller says this loudly and does not take the send for failed. -type SendNotRecordedError struct{ Err error } - -func (e *SendNotRecordedError) Error() string { - return "the record of the send was not written: " + e.Err.Error() -} -func (e *SendNotRecordedError) Unwrap() error { return e.Err } - -// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and -// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue -// 234), in one transaction. The digest and the generation are written together, so the would-send is never -// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if -// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest -// sends beyond the newest 200 for the machine are let go. -func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64, - generation int64, s Send) error { - var sentEpoch *int64 - if epoch > 0 { - e := int64(epoch) - sentEpoch = &e - } - tx, err := i.store.Pool().Begin(ctx) - if err != nil { - return err - } - defer func() { _ = tx.Rollback(ctx) }() - if _, err := tx.Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3, sent_generation = $4, - sent_provisioning = $5::jsonb - where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil { - return err - } - recordErr := recordSend(ctx, tx, node, s) - if err := tx.Commit(ctx); err != nil { - return err - } - // And what it put on the machine, in the assignments (issues 1095, 1109): a write of it that fails fails this record, - // for what a machine was sent is read from there. - if err := i.recordRuns(ctx, node, builds, time.Now()); err != nil { - return fmt.Errorf("%s was sent its declaration, and what each module of it runs there could not be written: %w", node, err) - } - if recordErr != nil { - return &SendNotRecordedError{Err: recordErr} - } - return nil -} - -// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written. -func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error { - sp, err := tx.Begin(ctx) - if err != nil { - return err - } - defer func() { _ = sp.Rollback(ctx) }() - modules := s.Modules - if modules == nil { - modules = []string{} - } - if _, err := sp.Exec(ctx, - `insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules) - values ($1, $2, $3, $4, $5, $6, $7)`, - node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest, - modules); err != nil { - return err - } - if err := pruneSends(ctx, sp, node); err != nil { - return err - } - return sp.Commit(ctx) -} - -// pruneSends lets go of a machine's sends beyond the newest 200. -func pruneSends(ctx context.Context, q queries, node string) error { - _, err := q.Exec(ctx, - `delete from declaration_send where node = $1 and id not in - (select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept) - return err -} - -// sendColumns are a send's columns as scanSends reads them. -const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0), - s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0), - s.recorded` - -func scanSends(rows pgx.Rows) ([]Send, error) { - defer rows.Close() - var out []Send - for rows.Next() { - var s Send - if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, - &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil { - return nil, err - } - out = append(out, s) - } - return out, rows.Err() -} - -// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false -// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it. -func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { - sends, err := i.lastSends(ctx, `n.name = $1`, name) - if err != nil { - return Send{}, false, err - } - s, found := sends[name] - return s, found, nil -} - -// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what -// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry. -func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) { - return i.lastSends(ctx, `true`) -} - -// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report -// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and: -// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's -// own. A periodic report about an identical earlier send names the same digest and may arrive after a later -// send the machine never received; only the sequence tells them apart. It also holds for a fast machine -// whose report lands before the send's row is stamped. -// - from an engine that reports no order, the report came after the send — both times the store's own — so -// a machine sent again a declaration it once applied does not read as having applied the new send. -func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) { - rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`, - coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at - from declaration_send s join node n on n.id = s.node - left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest - and case when r.reported_sequence is not null - then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0) - else r.at >= s.sent_at end - where s.recorded and `+where+` - order by n.name, s.id desc`, args...) - if err != nil { - return nil, err - } - defer rows.Close() - out := map[string]Send{} - for rows.Next() { - var s Send - if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, - &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded, - &s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil { - return nil, err - } - out[s.NodeName] = s - } - return out, rows.Err() -} - -// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and -// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal -// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may -// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none. -func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) { - rows, err := i.store.Pool().Query(ctx, `with refused as ( - update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4 - where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded - order by id desc limit 1) - returning *) - select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied, - nullIfZero(raisedTo)) - if err != nil { - return Send{}, false, err - } - sends, err := scanSends(rows) - if err != nil || len(sends) == 0 { - return Send{}, false, err - } - return sends[0], true, nil -} - -// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its -// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can -// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the -// refused declaration carried, and the refusal's applied generation and counter raise. -func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) { - rows, err := i.store.Pool().Query(ctx, `with refused as ( - insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied, - counter_raised_to, recorded) - values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *) - select `+sendColumns+` from refused s join node n on n.id = s.node`, - s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest, - s.RefusedApplied, nullIfZero(s.CounterRaisedTo)) - if err != nil { - return Send{}, err - } - sends, err := scanSends(rows) - if err != nil { - return Send{}, err - } - if len(sends) == 0 { - return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence) - } - if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil { - return Send{}, err - } - return sends[0], nil -} - -// RefusedSendsSince is every send refused for its generation since a moment, newest first. -func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) { - rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` - from declaration_send s join node n on n.id = s.node - where s.refused_at >= $1 order by s.refused_at desc`, since) - if err != nil { - return nil, err - } - return scanSends(rows) -} - -// nullIfEmpty is raw JSON, or null for none. -func nullIfEmpty(raw []byte) *string { - if len(raw) == 0 { - return nil - } - text := string(raw) - return &text -} diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go deleted file mode 100644 index f36cf94a..00000000 --- a/internal/inventory/sends_test.go +++ /dev/null @@ -1,342 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "slices" - "testing" - "time" -) - -// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration -// newer in sequence than every other named four fewer modules than the assignments held, a machine applied -// it, and nothing the mesh kept could say who sent it or from what view. - -func generationNow(t *testing.T, inv *Inventory) int64 { - t.Helper() - g, err := inv.AssignmentGeneration(t.Context()) - if err != nil { - t.Fatal(err) - } - return g -} - -// **Every change to what is assigned where raises the generation** — an assignment and an unassignment — and a repeated -// assignment, which changes nothing, does not. -func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) { - inv, node := aNodeWithModules(t, "postgres", "web") - ctx := t.Context() - start := generationNow(t, inv) - if start < 1 { - t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start) - } - if _, err := inv.Assign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - assigned := generationNow(t, inv) - if assigned <= start { - t.Fatalf("an assignment left the generation at %d", assigned) - } - if _, err := inv.Assign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - if again := generationNow(t, inv); again != assigned { - t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again) - } - if err := inv.Unassign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - unassigned := generationNow(t, inv) - if unassigned <= assigned { - t.Fatalf("an unassignment left the generation at %d", unassigned) - } - // A machine's removal, which no verb makes, is read from the machines the store knows: an assignment on a machine it - // does not know is none (assignments in the bucket have no key of the store to cascade through). -} - -// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter -// is behind it** — a store put back from a backup — so the next send is not refused for ever. -func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) { - inv, _ := aNodeWithModules(t) - now := generationNow(t, inv) - if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 { - t.Fatalf("raised past %d to %d (%v)", now+40, raised, err) - } - if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 { - t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err) - } -} - -// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it -// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation -// written with the digest, and a refusal is kept on the send it refused, naming its sender. -func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", - Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}} - if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil { - t.Fatal(err) - } - stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", - Generation: 38, Digest: "d12", Modules: []string{"docker"}} - if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil { - t.Fatal(err) - } - last, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - t.Fatalf("the last send is not kept: %v", err) - } - if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded || - !slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() { - t.Fatalf("the last send reads %+v", last) - } - if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 { - t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err) - } - - refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0) - if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 || - refused.CounterRaisedTo != 0 { - t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err) - } - since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 { - t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) - } - if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found { - t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err) - } -} - -// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it; -// it is not the machine's last send, and it says what the counter was raised to. -func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41, - Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45}) - if err != nil { - t.Fatal(err) - } - if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 || - kept.NodeName != node { - t.Fatalf("the refusal reads %+v", kept) - } - since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(since) != 1 || since[0].Sequence != 99 { - t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) - } - if _, found, err := inv.LastSend(ctx, node); err != nil || found { - t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err) - } -} - -// **The machine's record and the send's are written together, and a send record that cannot be written -// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a -// send that is away must not read as failed, nor the machine as behind. -func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - // A NUL byte is text PostgreSQL refuses: the send's own row cannot be written. - broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"} - err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken) - var unrecorded *SendNotRecordedError - if !errors.As(err, &unrecorded) { - t.Fatalf("a send record that could not be written read as %v", err) - } - if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 { - t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err) - } - if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" { - t.Fatalf("the machine's digest was not written: %q (%v)", sent, err) - } -} - -// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. -func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { - inv, node := aNodeWithModules(t) - record, err := inv.NodeByName(t.Context(), node) - if err != nil { - t.Fatal(err) - } - for _, reads := range []bool{false, true, false} { - if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil { - t.Fatal(err) - } - if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads { - t.Fatalf("recorded %v, read %v (%v)", reads, got, err) - } - } -} - -// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its -// words, or not answered yet — read from the machine's report only when that report is about the send's -// declaration and came after it, so a report about an earlier send is never this one's answer. And every -// machine's last send is read at once, for status, by name; a machine never sent anything has none. -func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - if _, err := inv.AddNode(ctx, "laptop"); err != nil { - t.Fatal(err) - } - send := func(sequence int64, digest string) { - t.Helper() - if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57, - Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest, - Modules: []string{"docker", "sudo"}}); err != nil { - t.Fatal(err) - } - } - report := func(d Doing) { - t.Helper() - if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil { - t.Fatal(err) - } - } - answer := func() SendAnswer { - t.Helper() - last, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - t.Fatalf("the last send is not read back: %v, %v", found, err) - } - return last.Answer - } - - send(11, "d11") - if a := answer(); a.Outcome != "" || a.At != nil { - t.Fatalf("a send not reported on reads as answered: %+v", a) - } - report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3}) - if a := answer(); a.Outcome != OutcomeApplied || a.At == nil { - t.Fatalf("a send the machine reported applying reads %+v", a) - } - - send(12, "d12") - if a := answer(); a.Outcome != "" { - t.Fatalf("the report about the send before is taken for this one's answer: %+v", a) - } - report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"}) - if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" { - t.Fatalf("a send the machine refused reads %+v", a) - } - - // Sent again a declaration it reported applying before the send: not answered until it reports again. - report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3}) - send(13, "d12") - if a := answer(); a.Outcome != "" { - t.Fatalf("a report from before the send is taken for its answer: %+v", a) - } - - all, err := inv.LastSends(ctx) - if err != nil { - t.Fatal(err) - } - if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 || - !slices.Equal(all[node].Modules, []string{"docker", "sudo"}) { - t.Fatalf("every machine's last send reads %+v", all) - } - if _, found := all["laptop"]; found { - t.Error("a machine never sent anything has a last send") - } -} - -// **A report about another declaration, or another send of the same one, is not the last send's answer** -// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives -// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about -// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only -// the report's epoch and sequence say which send it is about. -func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - send := func(sequence int64, digest string) { - t.Helper() - if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57, - Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil { - t.Fatal(err) - } - } - ordered := func(epoch, sequence int64, digest string) { - t.Helper() - if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1}, - ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil { - t.Fatal(err) - } - } - answer := func() string { - t.Helper() - last, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - t.Fatalf("the last send is not read back: %v, %v", found, err) - } - return last.Answer.Outcome - } - - // An engine that reports no order: the digest decides. - send(11, "d11") - send(12, "d12") - if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil { - t.Fatal(err) - } - if a := answer(); a != "" { - t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a) - } - - // An ordered engine: the epoch and sequence decide, whatever the digest and the time. - send(20, "d20") - ordered(57, 20, "d20") - if a := answer(); a != OutcomeApplied { - t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a) - } - send(21, "d20") - ordered(57, 20, "d20") - if a := answer(); a != "" { - t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a) - } - ordered(56, 21, "d20") - if a := answer(); a != "" { - t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a) - } - ordered(57, 21, "d20") - if a := answer(); a != OutcomeApplied { - t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a) - } -} - -// 1145/AC3: whether a machine reads modules is its latest word, as whether it reads a generation is. -func TestWhetherAMachineReadsModulesIsItsLatestWord(t *testing.T) { - inv, node := aNodeWithModules(t) - record, err := inv.NodeByName(t.Context(), node) - if err != nil { - t.Fatal(err) - } - for _, reads := range []bool{false, true, false} { - if err := inv.RecordReadsModules(t.Context(), record.ID, reads); err != nil { - t.Fatal(err) - } - if got, err := inv.ReadsModules(t.Context(), record.ID); err != nil || got != reads { - t.Fatalf("1145/AC3: recorded %v, read %v (%v)", reads, got, err) - } - } -} diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go deleted file mode 100644 index eeea1567..00000000 --- a/internal/inventory/sent_builds_test.go +++ /dev/null @@ -1,86 +0,0 @@ -//go:build integration - -package inventory - -import ( - "reflect" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// novox/hq issue 259: a send keeps which build of each module it carried. A machine never sent -// anything, or sent with nothing recorded — before this was kept, or by hand — is not known, which -// is not the same as having been sent nothing. -func TestASendKeepsTheBuildsItCarried(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - node, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil { - t.Fatalf("a machine never sent anything has known builds %v (%v): %v", builds, known, err) - } - - carried := map[string]string{"resolver": "abc123", "network": ""} - if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil { - t.Fatal(err) - } - // What it was sent is read from the keys of the machine's assignments that name a commit (issue 1109): a module carried - // with no commit known names none. - builds, known, err := inv.SentBuilds(ctx, "anchor") - if err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"resolver": "abc123"}) { - t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err) - } - - // Sent with nothing carried: known, and empty. - if err := inv.RecordSent(ctx, node.ID, "d2", map[string]string{}); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || !known || len(builds) != 0 { - t.Fatalf("an empty send: %v %v %v", builds, known, err) - } - - // Sent by hand: not known, and the digest still recorded. - if err := inv.RecordSent(ctx, node.ID, "d3", nil); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil { - t.Fatalf("a send whose builds are not known read as %v %v: %v", builds, known, err) - } - if sent, err := inv.Outstanding(ctx, "anchor"); err != nil || sent != "d3" { - t.Fatalf("the digest was not recorded with it: %q %v", sent, err) - } - - if _, known, err := inv.SentBuilds(ctx, "nobody"); err != nil || known { - t.Fatalf("a machine the mesh does not know: %v %v", known, err) - } -} - -// A module's current build is the commit its manifest was read at, with its upgrade policy. -func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "resolver", Version: "1"}, - Source{Repository: "novox/mesh-catalog", BuiltFrom: "c1"}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "by-hand", Version: "1"}, Source{}); err != nil { - t.Fatal(err) - } - if err := inv.SetUpgradeOf(ctx, "by-hand", Upgrade{RollOut: true}); err != nil { - t.Fatal(err) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - // Rolled out by default, one machine first and gated (novox/hq ADR 0236). - if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true, From: catalogue.FromDefault}) { - t.Errorf("resolver is at %+v", got) - } - if got := current["by-hand"]; got != (CurrentBuild{RollOut: true, From: catalogue.FromPerson}) { - t.Errorf("a module with no source is at %+v", got) - } -} diff --git a/internal/inventory/sent_bus_users_test.go b/internal/inventory/sent_bus_users_test.go deleted file mode 100644 index 33439454..00000000 --- a/internal/inventory/sent_bus_users_test.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build integration - -package inventory - -import "testing" - -// novox/hq issue 249: the digest of the user list a machine was last sent is kept, by its name, and -// is empty for a machine never sent one. -func TestTheUserListAMachineWasSentIsKept(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "" { - t.Fatalf("a machine never sent a list has %q: %v", sent, err) - } - if err := inv.RecordSentBusUsers(ctx, "anchor", "abc"); err != nil { - t.Fatal(err) - } - if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "abc" { - t.Fatalf("the list sent was not kept: %q %v", sent, err) - } - if sent, err := inv.SentBusUsers(ctx, "nobody"); err != nil || sent != "" { - t.Fatalf("a machine the mesh does not know: %q %v", sent, err) - } -} diff --git a/internal/inventory/sent_plans_test.go b/internal/inventory/sent_plans_test.go deleted file mode 100644 index fb1a081c..00000000 --- a/internal/inventory/sent_plans_test.go +++ /dev/null @@ -1,95 +0,0 @@ -//go:build integration - -package inventory - -import ( - "encoding/json" - "reflect" - "strings" - "testing" -) - -// A send keeps the provisioner's plans it handed (design 57 P6, issue 672, migration 0100): the assignments mirror -// carries the plan of the build that machine was sent, in the module's own assignment and no other, and a walk -// reads which templates the send handed. -func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) { - inv, node := aNodeWithModules(t, "dunst", "web") - ctx := t.Context() - for _, m := range []string{"dunst", "web"} { - if _, err := inv.Assign(ctx, node, m); err != nil { - t.Fatal(err) - } - } - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - digest := "sha256:" + strings.Repeat("d", 64) - plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, - Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", - Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} - // A plan for a module the machine no longer has assigned is kept with the send and mirrored nowhere; one for a - // module only pulled in by another is mirrored under its own key, saying what requires it (issue 765). - raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"gone":` + mustJSON(t, plan) + `,"notify":` + - strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { - t.Fatal(err) - } - - sent, _, err := inv.SentTo(ctx, node) - if err != nil || !reflect.DeepEqual(sent.Provisioned, map[string]string{"dunst": digest, "gone": digest, - "notify": digest}) { - t.Fatalf("the send says it handed %v (%v)", sent.Provisioned, err) - } - plans, err := inv.SentProvisioningPlans(ctx) - required := plan - required.RequiredBy = "desk" - if err != nil || !reflect.DeepEqual(plans, map[string]Provisioning{node + ".dunst": plan, node + ".notify": required}) { - t.Fatalf("the plans mirrored are %+v (%v)", plans, err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror := inv.PlanValuesForTest(t) - var dunst, web AssignmentMirrored - if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil { - t.Fatal(err) - } - if err := json.Unmarshal(mirror[node+".web"], &web); err != nil { - t.Fatal(err) - } - if dunst.Provisioning == nil || !reflect.DeepEqual(*dunst.Provisioning, plan) || web.Provisioning != nil { - t.Fatalf("the mirror carries %+v for dunst and %+v for web", dunst.Provisioning, web.Provisioning) - } - var notify AssignmentMirrored - if err := json.Unmarshal(mirror[node+".notify"], ¬ify); err != nil { - t.Fatalf("no value for the pulled-in module: %s (%v)", mirror[node+".notify"], err) - } - if notify.RequiredBy != "desk" || !notify.Assigned.IsZero() || notify.Provisioning == nil || - !reflect.DeepEqual(*notify.Provisioning, plan) { - t.Fatalf("the mirror carries %s for the pulled-in module", mirror[node+".notify"]) - } - if _, has := mirror[node+".gone"]; has { - t.Fatalf("a module no longer assigned is mirrored: %s", mirror[node+".gone"]) - } - - // The next send hands nothing: nothing is kept, and the assignment's value is the bytes it was. - if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second"}); err != nil { - t.Fatal(err) - } - if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil { - t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err) - } - if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") { - t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"]) - } else if _, has := mirror[node+".notify"]; has { - t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"]) - } -} - -func mustJSON(t *testing.T, v any) string { - t.Helper() - raw, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - return string(raw) -} diff --git a/internal/inventory/source.go b/internal/inventory/source.go index a498b07b..1b2ca3ae 100644 --- a/internal/inventory/source.go +++ b/internal/inventory/source.go @@ -30,7 +30,6 @@ type sourceRow struct { made []byte manifest []byte manifestDigest string - failedGate bool } // sourceRows is every successful build of each module (of one module, when named), newest request @@ -53,13 +52,10 @@ func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][ func standing(builds []sourceRow, from int) int { at := from fp := builds[from].fingerprint - if fp == "" || builds[from].failedGate { + if fp == "" { return at } for j := from + 1; j < len(builds); j++ { - if builds[j].failedGate { - break - } if builds[j].fingerprint != fp { if !newForm(fp) || newForm(builds[j].fingerprint) || !sameDigests(builds[at].made, builds[j].made) { break diff --git a/internal/inventory/stored_silent_file_test.go b/internal/inventory/stored_silent_file_test.go index 5840e6f3..cf664fce 100644 --- a/internal/inventory/stored_silent_file_test.go +++ b/internal/inventory/stored_silent_file_test.go @@ -39,9 +39,17 @@ func TestAStoredBuildWithASilentMergeableFileIsStillReadAndComposed(t *testing.T t.Fatal(err) } - m, found, err := inv.ManifestAt(ctx, "mail-client", b.Commit) - if err != nil || !found { - t.Fatalf("the stored build is not read again: found %v, %v", found, err) + c, err := inv.theChain(ctx) + if err != nil { + t.Fatal(err) + } + raw, err := inv.chainManifestOf(ctx, c, b.ID) // the manifest as the record keeps it, read from its digest when need be + if err != nil || len(raw) == 0 { + t.Fatalf("the stored build is not read again: %d bytes, %v", len(raw), err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("the stored manifest is not read again: %v", err) } if written := catalogue.WrittenSettingsProblems(m); len(written) != 1 { t.Fatalf("the rule this build predates should name its accounts file where it is written: %v", written) diff --git a/internal/inventory/tier_entered_resume_1886_test.go b/internal/inventory/tier_entered_resume_1886_test.go deleted file mode 100644 index f2263f09..00000000 --- a/internal/inventory/tier_entered_resume_1886_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package inventory - -import ( - "testing" - "time" -) - -// Issue 1886: R4e entered tier 1 at 02:01, was stopped, and was resumed at 07:43 at the same tier. Its save kept the -// tier's entry of 02:01, so at 07:50 the plan read "waiting for mesh-build-image on shanks to be applied for 5h48m53s" -// and its progress watchdog measured the tier from before the stop. A walk set open again at the tier it ended at -// enters that tier anew: its time in the tier runs from the resume. -func TestAWalkResumedAtItsTierEntersItAnew(t *testing.T) { - entered := time.Date(2026, 10, 23, 0, 1, 7, 0, time.UTC) - resumed := time.Date(2026, 10, 23, 5, 43, 30, 0, time.UTC) - old := &Plan{ID: "plan-r4e", State: PlanFailed, Tier: 1, Tiers: [][]string{{"a"}, {"b"}, {"c"}}, - Created: entered.Add(-2 * time.Hour), TierEntered: entered} - p := *old - p.State = PlanBuilding - got, d := tierLeftOf(old, p, resumed) - if !got.Equal(resumed) || d != nil { - t.Fatalf("1886: a walk resumed at tier 1 entered it at %s (duration %v); want %s, no tier left", got, d, resumed) - } - // A walk still in its tier keeps its entry; one that ended keeps it too. - open := *old - open.State = PlanRolling - if got, _ := tierLeftOf(&open, open, resumed); !got.Equal(entered) { - t.Fatalf("an open walk still in its tier was stamped anew: %s", got) - } - if got, _ := tierLeftOf(old, *old, resumed); !got.Equal(entered) { - t.Fatalf("a failed walk saved again was stamped anew: %s", got) - } -} diff --git a/internal/inventory/unseen.go b/internal/inventory/unseen.go index 023fa315..7fecb8e5 100644 --- a/internal/inventory/unseen.go +++ b/internal/inventory/unseen.go @@ -136,22 +136,3 @@ func (i *Inventory) layerNode(ctx context.Context, nodeName string) (*string, er } return &n.ID, nil } - -// RecordSentSummary keeps what a machine was last sent, summarised (migration 0078): read only to -// compare what would be sent with it, never as what the machine should be. -func (i *Inventory) RecordSentSummary(ctx context.Context, node string, summary []byte) error { - _, err := i.store.Pool().Exec(ctx, `update node set sent_summary = $2 where id = $1`, node, summary) - return err -} - -// SentSummary is what a machine was last sent, summarised, by its name; empty for a machine sent -// nothing since the summary was first kept. -func (i *Inventory) SentSummary(ctx context.Context, name string) ([]byte, error) { - var raw []byte - err := i.store.Pool().QueryRow(ctx, - `select sent_summary from node where name = $1`, name).Scan(&raw) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - return raw, err -} diff --git a/internal/inventory/unseen_test.go b/internal/inventory/unseen_test.go index 80ea3998..dc786660 100644 --- a/internal/inventory/unseen_test.go +++ b/internal/inventory/unseen_test.go @@ -83,24 +83,3 @@ func TestTheLayerASetReplacesIsKeptAndReadable(t *testing.T) { t.Fatalf("the node's history changed with the mesh's: %v %v", past, err) } } - -// What a machine was last sent is kept, summarised, and read back by its name; a machine sent -// nothing since has nothing to compare with. -func TestWhatAMachineWasSentIsKeptForComparison(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - n, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if got, err := inv.SentSummary(ctx, "anchor"); err != nil || len(got) != 0 { - t.Fatalf("a machine never sent anything: %s %v", got, err) - } - if err := inv.RecordSentSummary(ctx, n.ID, []byte(`[{"id":"web.server","type":"container"}]`)); err != nil { - t.Fatal(err) - } - got, err := inv.SentSummary(ctx, "anchor") - if err != nil || len(got) == 0 { - t.Fatalf("read back: %s %v", got, err) - } -} diff --git a/internal/inventory/waiting_merges_test.go b/internal/inventory/waiting_merges_test.go deleted file mode 100644 index 4601e937..00000000 --- a/internal/inventory/waiting_merges_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package inventory - -import ( - "context" - "testing" - "time" -) - -// A merge no plan answers waits, and is listed as waiting until it is answered where it is kept (novox/hq issue 1146): -// what a first merge into a new repository does while it waits for mesh-delivery's word. -func TestAMergeNoPlanAnswersIsListedWaitingUntilItIsAnswered(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - now := time.Now().UTC() - if added, err := inv.chainAddMerge(ctx, c, BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: "c1", - Merged: now, Heard: now}); err != nil || !added { - t.Fatalf("added %v, %v", added, err) - } - waiting, err := inv.chainWaitingMerges(ctx, c) - if err != nil || len(waiting) != 1 || waiting[0].Commit != "c1" { - t.Fatalf("waiting %+v, %v", waiting, err) - } - move := []MergeMove{{Repository: "novox/mesh-delivery", Commit: "c1", Plan: WaitingRecord}} - if err := inv.chainSavePlans(ctx, c, nil, move); err != nil { - t.Fatal(err) - } - if waiting, err = inv.chainWaitingMerges(ctx, c); err != nil || len(waiting) != 0 { - t.Fatalf("answered, it still waits: %+v, %v", waiting, err) - } - if _, known, err := inv.chainMergeOf(ctx, c, "novox/mesh-delivery", "c1"); err != nil || !known { - t.Fatalf("an answered merge is no longer kept: %v %v", known, err) - } -} diff --git a/internal/inventory/walkphases.go b/internal/inventory/walkphases.go deleted file mode 100644 index e7d42a45..00000000 --- a/internal/inventory/walkphases.go +++ /dev/null @@ -1,676 +0,0 @@ -package inventory - -import ( - "context" - "fmt" - "sort" - "strings" - "time" -) - -// Where a walk's time went (novox/hq ADR 0282 decision 6, issue 382): from its merge to every machine of its -// rest running its builds, phase by phase, worked out from the walk's own moments. Measurement only: nothing -// here decides anything about the walk. -// -// **A phase that cannot be measured is said unknown, never zero.** Where a moment is missing (a walk kept -// before it was recorded, a machine not yet reported), the phases around it are unknown, and the span between -// the moments on either side is counted as unknown time: the measured phases and the unknown time always add -// up to the total. A phase that did not happen (no window for a merge walked alone, no first machine for a -// module nobody runs) is said none, with no time. - -// The phases, in the order a walk passes them (ADR 0282's table). -const ( - PhaseWindow = "window" // the merge to its batch's window closing - PhaseQueued = "queued" // the window closed to the walk being cut: waiting behind another walk - PhaseWord = "word" // the cut to the delivery's word, for a walk that waits for one - PhaseBetween = "between-tiers" // one tier's end to the next tier's ask - PhaseBuild = "build" // a tier asked to its last module built - PhaseSend = "send-first" // built to sent to the first machines - PhaseBusy = "busy-wait" // sent first to a busy container there replaced, when one waited (ADR 0335) - PhaseJudge = "judgement" // sent first to the first-node gate's last verdict: its readings - PhaseRest = "send-rest" // judged to sent to the rest - PhaseApply = "apply" // the last send to every machine of the rest reporting the build applied - PhaseOpen = "open" // a walk still running: since its last moment - PhaseMeasured = "measured" - PhaseUnknown = "unknown" - PhaseNone = "none" -) - -// The classes of a walk (ADR 0282 decision 1) and their delivery budgets. -const ( - ClassCore = "core" - ClassLeaf = "leaf" -) - -// ApplySilentAfter is how long after a send to the rest a machine that has said nothing is left out of the -// walk's end, as a machine not heard from (ADR 0282 decision 1: a sleeping laptop is listed, not waited for). -const ApplySilentAfter = 15 * time.Minute - -// WalkPhases is a walk's time, phase by phase. -type WalkPhases struct { - Class string `json:"class,omitempty"` - // From is the walk's earliest merge; End when its last phase ended: every machine of its rest reported the - // build applied. Nil End while that is not known. - From *time.Time `json:"from,omitempty"` - End *time.Time `json:"end,omitempty"` - // TotalMS is End − From; zero while either is unknown, Total its words. - TotalMS int64 `json:"total_ms,omitempty"` - Total string `json:"total,omitempty"` - // UnknownMS is the time within the walk no phase could be measured over. - UnknownMS int64 `json:"unknown_ms,omitempty"` - Phases []WalkPhase `json:"phases"` - Silent []string `json:"silent,omitempty"` - // Rest is each machine of the rest's own report of the send that carried the walk's build there (novox/hq ADR - // 0282 decision 6, issue 582): when it was sent, when it said, and what. - Rest []RestReport `json:"rest,omitempty"` - Said string `json:"said"` - Merges []MergeStart `json:"merges,omitempty"` -} - -// MergeStart is one merge the walk answers and when it was made: where that merge's delivery time starts. -type MergeStart struct { - Repository string `json:"repository"` - Commit string `json:"commit"` - Merged time.Time `json:"merged"` -} - -// WalkPhase is one phase of a walk. -type WalkPhase struct { - Name string `json:"name"` - // Tier is the tier a tier's phase belongs to; -1 for a phase of the walk. - Tier int `json:"tier"` - State string `json:"state"` - Start *time.Time `json:"start,omitempty"` - End *time.Time `json:"end,omitempty"` - TookMS int64 `json:"took_ms,omitempty"` - Took string `json:"took,omitempty"` - Said string `json:"said,omitempty"` -} - -// AppliedReport is a machine's first report, after a send, that it applied what it was sent. -type AppliedReport struct { - At time.Time - Outcome string - // Answered is, for a machine that waited for a busy container (novox/hq ADR 0335), its first report of the - // send; Busy how long it then waited; Waiting that it still waits, At being its first report then. - Answered time.Time - Busy time.Duration - Waiting bool -} - -// AppliedLookup answers a machine's report of the send the walk made to it: send is what the walk kept of it, and -// walkSent the walk's own record of its send to the rest; false when it has not reported. -type AppliedLookup func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) - -// RestReport is one machine of the rest and its own report of a send of the walk (novox/hq issue 582). -type RestReport struct { - Node string `json:"node"` - Modules []string `json:"modules"` - // Sent is when that machine was sent, by its own send time where the walk kept it; else the walk's record of - // its send to the rest, and SentSaid says so. - Sent time.Time `json:"sent"` - SentSaid string `json:"sent_said,omitempty"` - Sequence int64 `json:"sequence,omitempty"` - Reported *time.Time `json:"reported,omitempty"` - Took string `json:"took,omitempty"` - Outcome string `json:"outcome,omitempty"` - // Busy is how long it waited for a busy container before it ran the build (novox/hq ADR 0335): inside Took, - // never read as the machine being silent. - Busy string `json:"busy,omitempty"` - // State is applied, failed, waiting (not yet heard, within ApplySilentAfter), busy (it answered and waits for - // a busy container: never silent) or silent (not heard within it: listed, never moving the walk's end). - State string `json:"state"` - busy time.Duration -} - -// The states of a machine of the rest. -const ( - RestApplied = "applied" - RestFailed = "failed" - RestWaiting = "waiting" - RestBusy = "busy" - RestSilent = "silent" -) - -// point is one moment of the walk, ending the phase named. -type point struct { - phase string - tier int - at *time.Time - none bool // the phase did not happen - said string // why it is none, or unknown -} - -// Phases is the walk's time phase by phase, at now; applied answers the rest's reports (nil: none read). -func (p Plan) WalkPhases(now time.Time, applied AppliedLookup) *WalkPhases { - if p.Release != nil || p.Batch() { - return nil - } - w := &WalkPhases{} - if p.Times != nil { - w.Class = p.Times.Class - } - from := p.firstMerge(w) - if from != nil { - f := from.Truncate(time.Millisecond) - from = &f - } - if from == nil { - w.Said = "when its merge was made is not kept: its delivery time is unknown" - } else { - w.From = from - } - points := p.points(now, applied, w) - // Walk the moments: each known moment after a known one is a measured phase; a missing moment makes the - // phases up to the next known one unknown, and their span unknown time. - last := from - var pending []int - for _, pt := range points { - if pt.none { - w.Phases = append(w.Phases, WalkPhase{Name: pt.phase, Tier: pt.tier, State: PhaseNone, Said: pt.said}) - continue - } - ph := WalkPhase{Name: pt.phase, Tier: pt.tier, Said: pt.said} - if pt.at == nil { - ph.State = PhaseUnknown - w.Phases = append(w.Phases, ph) - pending = append(pending, len(w.Phases)-1) - continue - } - at := pt.at.Truncate(time.Millisecond) - ph.End = &at - if last != nil && at.Before(*last) { - // **Out of order** (issue 382): this moment was recorded before the one before it — a module's first - // send kept before its build was, two clocks, a late record. Neither phase can be measured honestly: - // the one before it is said unknown and its time joins the unknown time, and this one is said unknown - // with no time. The walk goes on from the later moment, so nothing is negative and the sum holds. - for i := len(w.Phases) - 1; i >= 0; i-- { - prev := &w.Phases[i] - if prev.End == nil { - continue // a phase that did not happen, or is unknown with no moment: the one before carries last - } - if prev.State == PhaseMeasured { - w.UnknownMS += prev.TookMS - } - prev.State, prev.Start, prev.TookMS, prev.Took = PhaseUnknown, nil, 0, "" - prev.Said = "out of order: the phase after it ended first" - break - } - ph.State, ph.Said = PhaseUnknown, "out of order: it ended before the phase before it" - w.Phases = append(w.Phases, ph) - pending = nil - continue - } - if last != nil && len(pending) == 0 { - start := *last - ph.State, ph.Start = PhaseMeasured, &start - ph.TookMS = at.Sub(start).Milliseconds() - ph.Took = words(at.Sub(start)) - } else { - ph.State = PhaseUnknown - if last != nil { - w.UnknownMS += at.Sub(*last).Milliseconds() - } - } - w.Phases = append(w.Phases, ph) - pending = nil - last = &at - } - if len(pending) > 0 { - // The walk's last moments are unknown: it has no end. - if w.Said == "" { - w.Said = "its end is unknown: " + w.Phases[pending[0]].Name + " " + orNot(w.Phases[pending[0]].Said) - } - return w - } - if last == nil || from == nil { - return w - } - if p.Open() { - since := *last - w.Phases = append(w.Phases, WalkPhase{Name: PhaseOpen, Tier: -1, State: PhaseOpen, Start: &since, - TookMS: now.Sub(since).Milliseconds(), Took: words(now.Sub(since)), Said: "the walk is " + p.State}) - w.Said = "open: " + p.State + ", " + words(now.Sub(*from)) + " since its merge" - return w - } - if p.State != PlanDone { - w.Said = "ended " + p.State + ": no delivery time" - return w - } - end := *last - w.End = &end - w.TotalMS = end.Sub(*from).Milliseconds() - w.Total = words(end.Sub(*from)) - if w.Said == "" { - if w.UnknownMS > 0 { - w.Said = fmt.Sprintf("%s from its merge to running everywhere, %s of it unknown", w.Total, - words(time.Duration(w.UnknownMS)*time.Millisecond)) - } else { - w.Said = w.Total + " from its merge to running everywhere" - } - } - return w -} - -// firstMerge is when the walk's earliest merge was made, and every merge's start kept on w. -func (p Plan) firstMerge(w *WalkPhases) *time.Time { - var first *time.Time - if p.Delivery != nil { - for _, m := range p.Delivery.Merges { - if m.Merged.IsZero() { - continue - } - w.Merges = append(w.Merges, MergeStart{Repository: m.Repository, Commit: m.Commit, Merged: m.Merged}) - if first == nil || m.Merged.Before(*first) { - at := m.Merged - first = &at - } - } - } - if first == nil { - for _, c := range p.Carried() { - if c.Merged.IsZero() { - continue - } - w.Merges = append(w.Merges, MergeStart{Repository: c.Repository, Commit: c.Commit, Merged: c.Merged}) - if first == nil || c.Merged.Before(*first) { - at := c.Merged - first = &at - } - } - } - return first -} - -// points are the walk's moments in order. -func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []point { - var out []point - // The window and the wait behind another walk. - cut := p.Created - if p.Times != nil && p.Times.Cut != nil { - cut = *p.Times.Cut - } - switch { - case p.Times == nil: - out = append(out, point{phase: PhaseWindow, tier: -1, said: "not kept for a walk made before ADR 0282"}, - point{phase: PhaseQueued, tier: -1, at: &cut, said: "the window and the wait behind another walk together"}) - case p.Times.WindowClosed == nil: - out = append(out, point{phase: PhaseWindow, tier: -1, none: true, said: "no window: walked on its own"}, - point{phase: PhaseQueued, tier: -1, at: &cut}) - default: - closed := *p.Times.WindowClosed - out = append(out, point{phase: PhaseWindow, tier: -1, at: &closed}, point{phase: PhaseQueued, tier: -1, at: &cut}) - } - if p.Delivery != nil && p.Delivery.Awaits != "" { - out = append(out, point{phase: PhaseWord, tier: -1, at: p.Delivery.Go, said: "waiting for " + p.Delivery.Awaits + "'s word"}) - } else { - out = append(out, point{phase: PhaseWord, tier: -1, none: true, said: "waits for no word"}) - } - var lastSends []restSend - for t, tier := range p.Tiers { - if t > p.Tier || (t == p.Tier && p.Tier < len(p.Tiers) && !askedAnyOf(p, tier)) { - break - } - var asked, built, first, judged, rest, busyEnded *time.Time - allBuilt, anyFirst, allJudged, allRest := true, false, true, true - // A busy container on a first machine holds the judging until it is replaced (novox/hq ADR 0335 §4): a - // phase of its own, only in a tier where one waited, counted in the delivery time and never exempted. - anyBusy, allBusyEnded, busyOn := false, true, "" - for _, m := range tier { - s := p.Modules[m] - if s == nil { - allBuilt, allRest = false, false - continue - } - asked = earliest(asked, s.AskedAt) - if s.BuiltAt == nil { - if s.State != "deleted" { - allBuilt = false - } - } else { - built = latest(built, s.BuiltAt) - } - if s.FirstAt != nil { - anyFirst = true - first = earliest(first, s.FirstAt) - if s.Gate != nil && s.Gate.Busy != nil { - anyBusy = true - if s.Gate.Busy.Ended == nil { - allBusyEnded, busyOn = false, s.Gate.Busy.Machine - } else { - busyEnded = latest(busyEnded, s.Gate.Busy.Ended) - } - } - if s.Gate != nil { - if s.Gate.JudgedAt == nil { - allJudged = false - } else { - judged = latest(judged, s.Gate.JudgedAt) - } - } - } - if s.SentAt == nil { - if s.State != "deleted" { - allRest = false - } - } else { - rest = latest(rest, s.SentAt) - for node, what := range s.Rest { - lastSends = append(lastSends, restSend{module: m, node: node, walkSent: *s.SentAt, what: what}) - } - } - } - if t > 0 { - out = append(out, point{phase: PhaseBetween, tier: t, at: asked}) - } - if !allBuilt { - built = nil - } - out = append(out, point{phase: PhaseBuild, tier: t, at: built}) - if anyFirst { - if !allJudged { - judged = nil - } - out = append(out, point{phase: PhaseSend, tier: t, at: first}) - if anyBusy { - said := "" - if !allBusyEnded { - busyEnded, said = nil, "waits for a busy container on "+busyOn - } - out = append(out, point{phase: PhaseBusy, tier: t, at: busyEnded, said: said}) - } - out = append(out, point{phase: PhaseJudge, tier: t, at: judged}) - } else { - out = append(out, point{phase: PhaseSend, tier: t, none: true, said: "no first machine: nothing to judge"}, - point{phase: PhaseJudge, tier: t, none: true, said: "no first machine: nothing to judge"}) - } - if !allRest { - rest = nil - } - out = append(out, point{phase: PhaseRest, tier: t, at: rest}) - } - if p.State != PlanDone { - return out - } - // Every machine of the rest running the build: its first report after the send, applied. - if p.Times == nil { - return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's sends are not kept for a walk made before ADR 0282"}) - } - if len(lastSends) == 0 { - // novox/hq issue 720: since ADR 0360 no gate reads a walk; its one machine's report is its run. - return append(out, point{phase: PhaseApply, tier: -1, none: true, - said: "no machine beyond the first: its one machine's report was its run"}) - } - if applied == nil { - return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's reports were not read"}) - } - // Each machine of the rest, by its own report of the send that carried the build there (novox/hq issue 582): - // never a report of a send made after it, which another walk's send may be. - reports := map[string]*RestReport{} - var order []string - for _, s := range lastSends { - at, said := s.walkSent, "" - if !s.what.At.IsZero() { - at = s.what.At - } else { - said = "the walk's record of its send to the rest: this machine's own send time was not kept" - } - key := s.node + "\x00" + s.what.Digest + "\x00" + at.Format(time.RFC3339Nano) - if r := reports[key]; r != nil { - r.Modules = appendOnce(r.Modules, s.module) - continue - } - rr := &RestReport{Node: s.node, Modules: []string{s.module}, Sent: at, SentSaid: said, Sequence: s.what.Sequence} - // Judged on the modules this walk sent there, not on every module the machine's declaration hands (ADR 0362). - var sentThere []string - for _, o := range lastSends { - if o.node == s.node && o.what.Digest == s.what.Digest && o.what.At.Equal(s.what.At) { - sentThere = append(sentThere, o.module) - } - } - if r, ok := applied(s.node, s.what.Only(sentThere...), s.walkSent); ok { - got := r.At - rr.Reported, rr.Outcome, rr.Took = &got, r.Outcome, words(r.At.Sub(at)) - if r.Busy > 0 { - rr.Busy = words(r.Busy) - rr.busy = r.Busy - } - if r.Waiting { - rr.Reported, rr.Took, rr.State = nil, "", RestBusy - } - } - reports[key] = rr - order = append(order, key) - } - // A machine sent twice by the walk (one tier's rest, then the next's) that said nothing of the first send has - // said it of the second, which carries the first's builds too: both are this walk's. - for _, k := range order { - rr := reports[k] - if rr.Reported != nil { - continue - } - for _, l := range order { - later := reports[l] - if later.Node == rr.Node && later.Sent.After(rr.Sent) && later.Reported != nil && - (rr.Reported == nil || later.Reported.Before(*rr.Reported)) { - got := *later.Reported - rr.Reported, rr.Outcome, rr.Took, rr.State = &got, later.Outcome, words(got.Sub(rr.Sent)), "" - } - } - } - var end *time.Time - var waiting, failed, busy []string - for _, k := range order { - rr := reports[k] - switch { - case rr.State == RestBusy: - // It answered and waits for a busy container: never silent, and the walk's end waits for it. - busy = appendOnce(busy, rr.Node) - case rr.Reported == nil && now.Sub(rr.Sent) > ApplySilentAfter, - rr.Reported != nil && rr.Reported.Sub(rr.Sent)-rr.busy > ApplySilentAfter: - rr.State = RestSilent - w.Silent = appendOnce(w.Silent, rr.Node) - case rr.Reported == nil: - rr.State = RestWaiting - waiting = appendOnce(waiting, rr.Node) - case rr.Outcome != OutcomeApplied: - rr.State = RestFailed - failed = appendOnce(failed, rr.Node+" ("+rr.Outcome+")") - default: - rr.State = RestApplied - end = latest(end, rr.Reported) - } - sort.Strings(rr.Modules) - w.Rest = append(w.Rest, *rr) - } - sort.SliceStable(w.Rest, func(a, b int) bool { - if w.Rest[a].Node != w.Rest[b].Node { - return w.Rest[a].Node < w.Rest[b].Node - } - return w.Rest[a].Sent.Before(w.Rest[b].Sent) - }) - switch { - case len(failed) > 0: - return append(out, point{phase: PhaseApply, tier: -1, said: "not applied on " + strings.Join(failed, ", ")}) - case len(waiting) > 0 || len(busy) > 0: - var said []string - if len(waiting) > 0 { - said = append(said, "waiting for "+strings.Join(waiting, ", ")+" to report it applied") - } - if len(busy) > 0 { - said = append(said, "waits for a busy container on "+strings.Join(busy, ", ")) - } - return append(out, point{phase: PhaseApply, tier: -1, said: strings.Join(said, "; ")}) - case end == nil: - return append(out, point{phase: PhaseApply, tier: -1, said: "no machine of the rest heard from: " + - strings.Join(w.Silent, ", ")}) - } - // A machine may have reported before the last tier ended: the walk ends at whichever is later. - for i := len(out) - 1; i >= 0; i-- { - if out[i].at != nil { - if out[i].at.After(*end) { - e := *out[i].at - end = &e - } - break - } - } - said := "" - if len(w.Silent) > 0 { - sort.Strings(w.Silent) - said = "not waited for, not heard from: " + strings.Join(w.Silent, ", ") - } - return append(out, point{phase: PhaseApply, tier: -1, at: end, said: said}) -} - -type restSend struct { - module, node string - walkSent time.Time - what SentDeclaration -} - -func askedAnyOf(p Plan, tier []string) bool { - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.AskedAt != nil { - return true - } - } - return false -} - -func earliest(a, b *time.Time) *time.Time { - if b == nil { - return a - } - if a == nil || b.Before(*a) { - t := *b - return &t - } - return a -} - -func latest(a, b *time.Time) *time.Time { - if b == nil { - return a - } - if a == nil || b.After(*a) { - t := *b - return &t - } - return a -} - -func appendOnce(to []string, s string) []string { - for _, x := range to { - if x == s { - return to - } - } - return append(to, s) -} - -func orNot(s string) string { - if s == "" { - return "is not known" - } - return "(" + s + ")" -} - -// words is a duration as a person reads it. -func words(d time.Duration) string { - if d < 0 { - return "-" + words(-d) - } - return d.Round(100 * time.Millisecond).String() -} - -// FirstAppliedAfter is a machine's report of the send a walk made to it: the controller's `apply` durations, which -// measure every send to its first report (to-be 45 Phase 0), each row starting at the machine's own send time and -// naming the declaration it was about. send is what the walk kept of that send; walkSent its own record of the -// send to the rest, made after every machine of the rest was sent (novox/hq issue 582). FirstReportOf chooses. -func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool, error) { - from, to := applyRowsBetween(send, walkSent) - rows, err := i.store.Pool().Query(ctx, - `select kind, ref, started, took_ms, detail from duration - where subject = $2 and ((kind = $1 and started >= $3 and started < $4) or (kind = $5 and started >= $3)) - order by started`, - DurationApply, node, from, to, DurationBusy) - if err != nil { - return AppliedReport{}, false, err - } - defer rows.Close() - var ds []Duration - for rows.Next() { - d := Duration{Subject: node, Node: node} - var ms int64 - if err := rows.Scan(&d.Kind, &d.Ref, &d.Started, &ms, &d.Detail); err != nil { - return AppliedReport{}, false, err - } - d.Took = time.Duration(ms) * time.Millisecond - ds = append(ds, d) - } - if err := rows.Err(); err != nil { - return AppliedReport{}, false, err - } - r, ok := FirstReportOf(ds, send, walkSent) - return r, ok, nil -} - -// applyRowsBetween is the span of a machine's `apply` durations that may hold its report of a walk's send. -func applyRowsBetween(send SentDeclaration, walkSent time.Time) (time.Time, time.Time) { - switch { - case !send.At.IsZero(): - return send.At.Add(-time.Millisecond), send.At.Add(time.Millisecond) - case send.Digest != "": - return walkSent.Add(-restSendBound), walkSent.Add(ApplySilentAfter) - } - return walkSent.Add(-appliedSlack), walkSent.Add(ApplySilentAfter) -} - -// FirstReportOf picks, from a machine's `apply` durations ordered by their start, its report of the send a walk made -// to it (novox/hq issue 582): -// - with the machine's own send time kept (every walk since issue 582): the duration of that very send, and of the -// declaration sent where the walk kept its digest. A report of a later send is never this walk's: a send made -// later, by another walk, would end this walk at that walk's send. -// - with only the declaration's digest kept (a walk kept before): the first duration about that declaration -// within restSendBound before the walk's record of its send, the machine sent first in the rest included. -// - with neither (a walk kept before ADR 0282's rest): the first duration started no earlier than appliedSlack -// before the walk's record of its send, as it was read before. -// -// A send past ApplySilentAfter is never read: its machine slept, and is listed silent (ADR 0282 decision 1). -// -// **A first report that said a container waits because it is busy** (novox/hq ADR 0335) is the machine's answer, -// and not yet the build running: its report is the end of its busy wait, the `busy-wait` duration of the same -// send, with the wait said apart (Busy). While none is kept the machine is still waiting (Waiting), never silent. -func FirstReportOf(ds []Duration, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - from, to := applyRowsBetween(send, walkSent) - for _, d := range ds { - if d.Kind == DurationBusy || d.Started.Before(from) || !d.Started.Before(to) { - continue - } - if send.Digest != "" && !strings.HasPrefix(d.Ref, send.Digest+"@") { - continue - } - answered := d.Started.Add(d.Took).UTC() - if d.Detail != ApplyBusy { - return AppliedReport{At: answered, Outcome: d.Detail}, true - } - for _, b := range ds { - if b.Kind == DurationBusy && b.Ref == d.Ref { - return AppliedReport{At: b.Started.Add(b.Took).UTC(), Outcome: OutcomeApplied, Answered: answered, - Busy: b.Took}, true - } - } - return AppliedReport{At: answered, Outcome: OutcomeApplied, Answered: answered, Waiting: true}, true - } - return AppliedReport{}, false -} - -// appliedSlack is how much earlier than a walk's record of its send the machine's own record of it may be, for a -// walk that kept neither the machine's send time nor what it sent. -const appliedSlack = 2 * time.Second - -// restSendBound is how long before a walk's record of its send to the rest a machine of the rest may have been -// sent, for a walk that kept what it sent and not when: the send to the rest goes machine by machine, and took -// 19 to 28 seconds on 2026-10-12 (novox/hq issue 582). -const restSendBound = 5 * time.Minute diff --git a/internal/inventory/walkphases_582_integration_test.go b/internal/inventory/walkphases_582_integration_test.go deleted file mode 100644 index 643d2510..00000000 --- a/internal/inventory/walkphases_582_integration_test.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "g14") - if err != nil { - t.Fatal(err) - } - if _, err := inv.NextSequence(ctx, record.ID); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil { - t.Fatal(err) - } - sent, found, err := inv.SentTo(ctx, "g14") - if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" { - t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil { - t.Fatal(err) - } - // The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send. - r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) { - t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err) - } - // A later send, reported: never this walk's. - if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil { - t.Fatal(err) - } - again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if !again.At.Equal(r.At) { - t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r) - } -} diff --git a/internal/inventory/walkphases_582_test.go b/internal/inventory/walkphases_582_test.go deleted file mode 100644 index 0b363f87..00000000 --- a/internal/inventory/walkphases_582_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package inventory - -import ( - "strings" - "testing" - "time" -) - -// novox/hq issue 582: the walk's own record of its send to the rest is made after every machine of the rest was -// sent, and that send took 19 to 28 s on 2026-10-12. The machine sent first reported 5 s after its own send, long -// before the walk's record; read from the walk's record, its report was missed and the walk ended at another walk's -// send to it, minutes later. A walk of one tier and three machines of the rest, the send to the rest taking 20 s. -type sendRest582 struct { - judged, walkSent time.Time - walk Plan - rows map[string][]Duration -} - -func walk582(t *testing.T) sendRest582 { - t.Helper() - judged := time.Date(2026, 10, 12, 17, 45, 35, 137000000, time.UTC) - ownSend := map[string]time.Duration{"g14": 500 * time.Millisecond, "novox": 10 * time.Second, "shanks": 20 * time.Second} - walkSent := judged.Add(20200 * time.Millisecond) - merged := judged.Add(-10 * time.Minute) - built, first := judged.Add(-3*time.Minute), judged.Add(-2*time.Minute) - rest := map[string]SentDeclaration{} - rows := map[string][]Duration{} - took := map[string]time.Duration{"g14": 5 * time.Second, "novox": 13 * time.Second, "shanks": 15 * time.Second} - for i, n := range []string{"g14", "novox", "shanks"} { - at := judged.Add(ownSend[n]) - digest := "walk-" + n - rest[n] = SentDeclaration{Digest: digest, Sequence: int64(647 + i), At: at} - rows[n] = append(rows[n], - // This walk's send, and its report. - Duration{Kind: DurationApply, Subject: n, Node: n, Ref: digest + "@" + at.Format("2006-01-02T15:04:05.000000"), - Started: at, Took: took[n], Detail: OutcomeApplied}, - // The next walk's send to the same machine, nine minutes later, and its report. - Duration{Kind: DurationApply, Subject: n, Node: n, Ref: "next-" + n + "@x", - Started: judged.Add(9*time.Minute + ownSend[n]), Took: 4 * time.Second, Detail: OutcomeApplied}) - } - p := Plan{ID: "plan-582", State: PlanDone, Tier: 1, Tiers: [][]string{{"build-agent"}}, Created: merged, - Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/mesh-controller", Commit: "c", Merged: merged}}}, - Times: &PlanTimes{Cut: &merged, Class: ClassCore}, - Modules: map[string]*PlanModule{"build-agent": {State: "built", AskedAt: &merged, BuiltAt: &built, - FirstAt: &first, SentAt: &walkSent, Gate: &PlanGate{JudgedAt: &judged}, Rest: rest}}} - return sendRest582{judged: judged, walkSent: walkSent, walk: p, rows: rows} -} - -// lookup answers from the rows as the controller's durations would. -func (w sendRest582) lookup(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - return FirstReportOf(w.rows[node], send, walkSent) -} - -// The apply phase ends at the last machine's own report of this walk's send — shanks, sent at the end of the 20 s -// and reporting 15 s later — and never at the next walk's send; every machine's report is shown. -func TestTheApplyPhaseEndsAtEachMachinesOwnReport(t *testing.T) { - c := walk582(t) - w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - want := c.judged.Add(35 * time.Second) // shanks: sent 20 s after the judgement, reported 15 s later - if !w.End.Equal(want) { - t.Fatalf("the walk ends at %s, want %s: its rest %+v", w.End, want, w.Rest) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != want.Sub(c.walkSent).Milliseconds() { - t.Fatalf("apply: %+v", got) - } - if len(w.Rest) != 3 || len(w.Silent) != 0 { - t.Fatalf("every machine of the rest is shown with its report: %+v, silent %v", w.Rest, w.Silent) - } - g14 := w.Rest[0] - if g14.Node != "g14" || g14.State != RestApplied || g14.Reported == nil || - !g14.Reported.Equal(c.judged.Add(5500*time.Millisecond)) || g14.Took != "5s" || g14.Sequence != 647 || - !g14.Sent.Equal(c.judged.Add(500*time.Millisecond)) || g14.SentSaid != "" { - t.Fatalf("g14, sent first and reporting 5 s after its own send: %+v", g14) - } -} - -// A walk kept before its machines' own send times were: matched by the declaration it sent, within restSendBound -// of its record of the send, so the machine sent first is still found; a walk that kept neither is read as before. -func TestAnOlderWalkIsReadByWhatItSent(t *testing.T) { - c := walk582(t) - for n, s := range c.walk.Modules["build-agent"].Rest { - s.At = time.Time{} - c.walk.Modules["build-agent"].Rest[n] = s - } - w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - if !w.End.Equal(c.judged.Add(35 * time.Second)) { - t.Fatalf("by the declaration sent: end %s, rest %+v", w.End, w.Rest) - } - if !strings.Contains(w.Rest[0].SentSaid, "walk's record") || !w.Rest[0].Sent.Equal(c.walkSent) { - t.Fatalf("a send time not kept is said: %+v", w.Rest[0]) - } - // Nothing of the send kept: the first report no earlier than appliedSlack before the walk's record, as before. - if r, ok := FirstReportOf(c.rows["shanks"], SentDeclaration{}, c.walkSent); !ok || !r.At.Equal(c.judged.Add(35*time.Second)) { - t.Fatalf("a walk that kept nothing of its send: %+v %v", r, ok) - } -} - -// A machine that said nothing of this walk's send is waiting, then silent past ApplySilentAfter: the report of a -// later send to it is never taken as this walk's, and never moves the walk's end. -func TestALaterSendsReportIsNeverThisWalks(t *testing.T) { - c := walk582(t) - c.rows["g14"] = c.rows["g14"][1:] // g14 slept through this walk's send; the next walk's send it reported - w := c.walk.WalkPhases(c.judged.Add(5*time.Minute), c.lookup) - if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") { - t.Fatalf("g14 not heard from yet: end %v, apply %+v", w.End, phase(w, PhaseApply, -1)) - } - w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - if len(w.Silent) != 1 || w.Silent[0] != "g14" || !w.End.Equal(c.judged.Add(35*time.Second)) { - t.Fatalf("g14 silent, the end at shanks' report: silent %v, end %s", w.Silent, w.End) - } - if w.Rest[0].State != RestSilent || w.Rest[0].Reported != nil { - t.Fatalf("g14's later report taken as this walk's: %+v", w.Rest[0]) - } -} - -// A machine sent by two tiers of one walk that said nothing of the first send has said it of the second, which -// carries the first tier's build too. -func TestAMachineSentTwiceByOneWalkIsHeardByItsLaterSend(t *testing.T) { - c := walk582(t) - second := c.judged.Add(4 * time.Minute) - secondSent := second.Add(time.Second) - c.walk.Tiers = append(c.walk.Tiers, []string{"route-proxy"}) - c.walk.Tier = 2 - asked, built := c.judged.Add(time.Minute), c.judged.Add(2*time.Minute) - first := built.Add(time.Second) - c.walk.Modules["route-proxy"] = &PlanModule{State: "built", AskedAt: &asked, BuiltAt: &built, FirstAt: &first, - SentAt: &secondSent, Gate: &PlanGate{JudgedAt: &second}, - Rest: map[string]SentDeclaration{"novox": {Digest: "walk-novox-2", Sequence: 1554, At: second.Add(500 * time.Millisecond)}}} - c.rows["novox"] = []Duration{{Kind: DurationApply, Subject: "novox", Ref: "walk-novox-2@x", - Started: second.Add(500 * time.Millisecond), Took: 14 * time.Second, Detail: OutcomeApplied}} - w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - want := second.Add(14500 * time.Millisecond) - if !w.End.Equal(want) || len(w.Silent) != 0 { - t.Fatalf("end %s (want %s), silent %v, rest %+v", w.End, want, w.Silent, w.Rest) - } - for _, r := range w.Rest { - if r.Node == "novox" && (r.State != RestApplied || !r.Reported.Equal(want)) { - t.Fatalf("novox, heard by its later send of this walk: %+v", r) - } - } -} - -// The store: a machine's own send time is read with what it was sent, and its `apply` duration is found by it — -// the send and the duration start at one moment, the node's sent_at. - -// Issue 840 (ADR 0362): the walk's report of a machine is asked for with only the provisioned modules the walk sent -// there, never every module the machine's declaration hands to the provisioner. -func TestTheRestIsAskedAboutTheModulesTheWalkSent(t *testing.T) { - c := walk582(t) - rest := c.walk.Modules["build-agent"].Rest - for n, s := range rest { - s.Provisioned = map[string]string{"build-agent": "t1", "other": "t2"} - rest[n] = s - } - asked := 0 - c.walk.WalkPhases(c.judged.Add(time.Hour), func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - asked++ - if len(send.Provisioned) != 1 || send.Provisioned["build-agent"] != "t1" { - t.Fatalf("%s is asked about %v", node, send.Provisioned) - } - return c.lookup(node, send, walkSent) - }) - if asked == 0 { - t.Fatal("no machine was asked") - } -} diff --git a/internal/inventory/walkphases_integration_test.go b/internal/inventory/walkphases_integration_test.go deleted file mode 100644 index f7cbb108..00000000 --- a/internal/inventory/walkphases_integration_test.go +++ /dev/null @@ -1,77 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := walkOf(t, recordedWalk) - p.Revision, p.Epoch = 0, 0 - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Class != ClassCore || back.Times.WindowClosed == nil || - !back.Times.WindowClosed.Equal(*p.Times.WindowClosed) { - t.Fatalf("the walk's moments were not kept: %v %+v", err, back.Times) - } - if back.Delivery.Merges[0].Merged.IsZero() { - t.Fatal("a merge's time was not kept") - } - sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) - if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - for _, d := range []Duration{ - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "old@1", Started: sent.Add(-time.Hour), Took: time.Second, Detail: OutcomeApplied}, - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "d1@2", Started: sent.Add(-time.Second), Took: 12 * time.Second, Detail: OutcomeApplied}, - } { - if err := inv.RecordDuration(ctx, d); err != nil { - t.Fatal(err) - } - } - r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent) - if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) { - t.Fatalf("the report after the send: %+v %v %v", r, ok, err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || anyKept || total { - t.Fatalf("nothing kept yet: %v %v %v", anyKept, total, err) - } - if err := inv.RecordDuration(ctx, Duration{Kind: DurationWalkPhase, Subject: "core/total", Ref: p.ID + "/total", - Started: sent, Took: time.Minute}); err != nil { - t.Fatal(err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || !anyKept || !total { - t.Fatalf("the total kept: %v %v %v", anyKept, total, err) - } -} - -func TestALateReportIsSilentNotTheEnd(t *testing.T) { - sent := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - p := walkOf(t, recordedWalk) - p.Modules["mesh-delivery"].SentAt = &sent - p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}} - w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - if node == "server" { - return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true - } - return AppliedReport{At: sent.Add(2 * time.Hour), Outcome: OutcomeApplied}, true - }) - if len(w.Silent) != 1 || w.Silent[0] != "laptop" || w.End == nil || !w.End.Equal(sent.Add(20*time.Second)) { - t.Fatalf("a late report: silent %v, end %v", w.Silent, w.End) - } - inv := ForTest(t) - ctx := t.Context() - if err := inv.RecordDuration(ctx, Duration{Kind: DurationApply, Subject: "laptop", Node: "laptop", Ref: "late@1", - Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil { - t.Fatal(err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err) - } -} diff --git a/internal/inventory/walkphases_test.go b/internal/inventory/walkphases_test.go deleted file mode 100644 index 216d6d5a..00000000 --- a/internal/inventory/walkphases_test.go +++ /dev/null @@ -1,263 +0,0 @@ -package inventory - -import ( - "encoding/json" - "strings" - "testing" - "time" -) - -// A walk recorded on the live mesh on 2026-10-10 (mesh-delivery, one tier, one machine), as `delivery walks` -// gave it, with the moments ADR 0282 adds: its window closed ninety seconds after its merge was heard, and it was -// cut eleven seconds later. -const recordedWalk = `{ - "id": "plan-1791654663505629616", "repository": "novox/mesh-catalog", "branch": "main", - "commit": "a14fa306f262d88bdcca83432a70df353d2eceb0", "merged": "2026-10-10T17:50:53Z", - "created": "2026-10-10T19:52:34.037755+02:00", "updated": "2026-10-10T19:55:37.974069+02:00", - "state": "done", "tier": 1, "tiers": [["mesh-delivery"]], - "modules": {"mesh-delivery": {"state": "built", - "asked_at": "2026-10-10T17:52:34.209423145Z", "built_at": "2026-10-10T17:52:50.818011314Z", - "sent_at": "2026-10-10T17:55:35.894985053Z", "first": ["novox"], "first_at": "2026-10-10T17:53:07.287136827Z", - "gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3, - "verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}}}, - "delivery": {"awaits": "", "merges": [{"repository": "novox/mesh-catalog", - "commit": "a14fa306f262d88bdcca83432a70df353d2eceb0", "number": 187, "merged": "2026-10-10T17:50:53Z"}]}, - "times": {"window_closed": "2026-10-10T17:52:23Z", "cut": "2026-10-10T17:52:34.037755Z", "class": "core"} -}` - -func walkOf(t *testing.T, raw string) Plan { - t.Helper() - var p Plan - if err := json.Unmarshal([]byte(raw), &p); err != nil { - t.Fatal(err) - } - return p -} - -// sumsUp checks the phases' measured time and the unknown time add up to the total, to the millisecond. -func sumsUp(t *testing.T, w *WalkPhases) { - t.Helper() - if w.End == nil || w.From == nil { - t.Fatalf("the walk has no end: %+v", w) - } - var sum int64 - for _, ph := range w.Phases { - if ph.State == PhaseMeasured { - if ph.Start == nil || ph.End == nil || ph.End.Sub(*ph.Start).Milliseconds() != ph.TookMS { - t.Errorf("phase %s %d says %dms between %v and %v", ph.Name, ph.Tier, ph.TookMS, ph.Start, ph.End) - } - sum += ph.TookMS - } - if ph.State == PhaseUnknown && ph.TookMS != 0 { - t.Errorf("an unknown phase %s says a time: %dms", ph.Name, ph.TookMS) - } - } - if sum+w.UnknownMS != w.TotalMS || w.End.Sub(*w.From).Milliseconds() != w.TotalMS { - t.Fatalf("the phases add up to %dms and %dms unknown, the total is %dms (%s): %+v", sum, w.UnknownMS, - w.TotalMS, w.End.Sub(*w.From), w.Phases) - } -} - -func phase(w *WalkPhases, name string, tier int) WalkPhase { - for _, ph := range w.Phases { - if ph.Name == name && ph.Tier == tier { - return ph - } - } - return WalkPhase{} -} - -// The phases of a recorded walk add up to its measured total: its merge at 17:50:53 to its verdict on its only -// machine at 17:55:35.894, 4m42.894s. -func TestARecordedWalksPhasesAddUpToItsTotal(t *testing.T) { - w := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - if w.TotalMS != 282894 || w.Class != ClassCore || w.UnknownMS != 0 { - t.Fatalf("total %dms (want 282894), class %q, unknown %d", w.TotalMS, w.Class, w.UnknownMS) - } - for name, want := range map[string]int64{PhaseWindow: 90000, PhaseQueued: 11037, PhaseBuild: 16781, - PhaseSend: 16469, PhaseJudge: 148607, PhaseRest: 0} { - tier := 0 - if name == PhaseWindow || name == PhaseQueued { - tier = -1 - } - if got := phase(w, name, tier); got.State != PhaseMeasured || got.TookMS != want { - t.Errorf("%s: %s %dms, want measured %dms", name, got.State, got.TookMS, want) - } - } - // The cut to the first ask (171ms) is a time no phase of ADR 0282's table names: it is counted in the build. - if got := phase(w, PhaseWord, -1); got.State != PhaseNone { - t.Errorf("a walk that waits for no word says its word phase %s", got.State) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseNone || !strings.Contains(got.Said, "no machine beyond the first: its one machine's report was its run") { - t.Errorf("one machine only: apply %s (%s)", got.State, got.Said) - } -} - -// Two tiers, a machine of the rest each, both reports read: every phase measured, the walk ends at the last -// report applied, and the phases add up. -func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) { - at := func(s string) *time.Time { - v, err := time.Parse(time.RFC3339Nano, "2026-10-10T18:"+s+"Z") - if err != nil { - t.Fatal(err) - } - return &v - } - p := Plan{ID: "plan-2", State: PlanDone, Tier: 2, Tiers: [][]string{{"a"}, {"b"}}, Created: *at("01:40"), - Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/x", Commit: "c1", Merged: *at("00:00")}, - {Repository: "novox/x", Commit: "c0", Merged: *at("00:30")}}}, - Times: &PlanTimes{WindowClosed: at("01:30"), Cut: at("01:40"), Class: ClassLeaf}, - Modules: map[string]*PlanModule{ - "a": {State: "built", AskedAt: at("01:41"), BuiltAt: at("02:05"), FirstAt: at("02:20"), SentAt: at("05:00"), - Gate: &PlanGate{JudgedAt: at("04:50")}, Rest: map[string]SentDeclaration{"ace": {Digest: "d1"}}}, - "b": {State: "built", AskedAt: at("05:10"), BuiltAt: at("05:40"), FirstAt: at("05:50"), SentAt: at("08:00.5"), - Gate: &PlanGate{JudgedAt: at("07:59")}, Rest: map[string]SentDeclaration{"shanks": {Digest: "d2"}}}, - }} - reports := map[string]AppliedReport{"ace": {At: *at("05:12"), Outcome: OutcomeApplied}, - "shanks": {At: *at("08:15.25"), Outcome: OutcomeApplied}} - w := p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - sumsUp(t, w) - if w.TotalMS != (8*time.Minute + 15250*time.Millisecond).Milliseconds() { - t.Fatalf("the walk's delivery time runs from its earliest merge to the last report: %s", w.Total) - } - if got := phase(w, PhaseBetween, 1); got.TookMS != 10000 { - t.Errorf("between the tiers: %dms", got.TookMS) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != 14750 { - t.Errorf("apply: %s %dms", got.State, got.TookMS) - } - if len(w.Merges) != 2 { - t.Errorf("each merge's start is said, for its own delivery time: %+v", w.Merges) - } - // A report not yet in: the walk has no end, and its apply is unknown — never zero. - delete(reports, "shanks") - w = p.WalkPhases(*at("10:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - if w.End != nil || w.TotalMS != 0 { - t.Fatalf("a walk whose rest has not reported has no end: %+v", w) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseUnknown || got.TookMS != 0 || !strings.Contains(got.Said, "shanks") { - t.Errorf("apply while shanks has not reported: %+v", got) - } - // Silent past the bound: listed, not waited for. - w = p.WalkPhases(at("08:00.5").Add(ApplySilentAfter+time.Second), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - sumsUp(t, w) - if len(w.Silent) != 1 || w.Silent[0] != "shanks" { - t.Errorf("a machine silent past the bound is said, not waited for: %+v", w.Silent) - } - // A failed report: the walk has no end, said. - reports["shanks"] = AppliedReport{At: *at("08:10"), Outcome: OutcomeFailed} - w = p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "shanks (failed)") { - t.Errorf("a failed apply ends nothing: %+v", phase(w, PhaseApply, -1)) - } -} - -// A moment that is missing makes the phases around it unknown, and their span unknown time: never a zero. -func TestAMissingMomentIsUnknownNeverZero(t *testing.T) { - p := walkOf(t, recordedWalk) - p.Modules["mesh-delivery"].BuiltAt = nil - w := p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - if got := phase(w, PhaseBuild, 0); got.State != PhaseUnknown || got.TookMS != 0 { - t.Errorf("a build without its moment: %+v", got) - } - if got := phase(w, PhaseSend, 0); got.State != PhaseUnknown { - t.Errorf("the send after a build without its moment: %+v", got) - } - if w.UnknownMS != 16781+16469 { - t.Errorf("the unknown time is the span between the moments around it: %dms", w.UnknownMS) - } - // A walk kept before ADR 0282: its window and its wait together, and its apply unknown. - p = walkOf(t, recordedWalk) - p.Times = nil - w = p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - if got := phase(w, PhaseWindow, -1); got.State != PhaseUnknown { - t.Errorf("a window not kept: %+v", got) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseUnknown || w.End != nil { - t.Errorf("a rest not kept: %+v, end %v", got, w.End) - } -} - -// An open walk is said open since its last moment, with no end. -func TestAnOpenWalkHasNoEnd(t *testing.T) { - p := walkOf(t, recordedWalk) - p.State, p.Tier = PlanRolling, 0 - p.Modules["mesh-delivery"].SentAt = nil - p.Modules["mesh-delivery"].Gate.JudgedAt = nil - w := p.WalkPhases(time.Date(2026, 10, 10, 17, 54, 0, 0, time.UTC), nil) - if w.End != nil || !strings.HasPrefix(w.Said, "its end is unknown") && !strings.HasPrefix(w.Said, "open") { - t.Fatalf("an open walk: %+v", w) - } - if got := phase(w, PhaseBuild, 0); got.State != PhaseMeasured { - t.Errorf("an open walk's phases so far are measured: %+v", got) - } -} - -// A gate keeps every pass and the first reading after one that did not pass, at most maxReadings. -func TestAGateKeepsItsReadings(t *testing.T) { - var g PlanGate - t0 := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - g.Read(t0, false, "starting") - g.Read(t0.Add(time.Second), false, "starting") - g.Read(t0.Add(40*time.Second), true, "") - g.Read(t0.Add(80*time.Second), true, "") - g.Read(t0.Add(81*time.Second), false, "gone") - g.Read(t0.Add(82*time.Second), false, "gone") - if len(g.Readings) != 4 || g.Readings[0].Said != "starting" || !g.Readings[2].Healthy || g.Readings[3].Said != "gone" { - t.Fatalf("readings: %+v", g.Readings) - } - for i := 0; i < 50; i++ { - g.Read(t0.Add(time.Duration(100+i)*time.Second), true, "") - } - if len(g.Readings) != maxReadings { - t.Fatalf("readings are bounded: %d", len(g.Readings)) - } -} - -// A walk's moments are kept and read back with it; a machine's first report after a send is read from the -// controller's apply durations. - -// A machine that slept past the bound and reported later is listed silent: its late report never moves the -// walk's end; and a report sent past the bound is not read as the walk's. - -// Moments out of order — #206's own walk on 2026-10-10 kept its first send 783 ms before its build — are said -// unknown, never a negative phase: the phase before is unknown and its time joins the unknown time, the phase -// out of order has no time, and the phases still add up to the total. -func TestMomentsOutOfOrderAreUnknownNeverNegative(t *testing.T) { - p := walkOf(t, recordedWalk) - built := time.Date(2026, 10, 10, 17, 53, 8, 70000000, time.UTC) // 783 ms after its first send - p.Modules["mesh-delivery"].BuiltAt = &built - w := p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - for _, ph := range w.Phases { - if ph.TookMS < 0 { - t.Fatalf("a negative phase: %+v", ph) - } - } - build, send := phase(w, PhaseBuild, 0), phase(w, PhaseSend, 0) - if build.State != PhaseUnknown || send.State != PhaseUnknown || send.TookMS != 0 || - !strings.Contains(send.Said, "out of order") || !strings.Contains(build.Said, "out of order") { - t.Fatalf("build %+v, send-first %+v", build, send) - } - // The build's span (cut to its later moment) is unknown time; the judgement is measured from the later moment. - if w.UnknownMS != built.Sub(time.Date(2026, 10, 10, 17, 52, 34, 37000000, time.UTC)).Milliseconds() { - t.Fatalf("unknown %dms", w.UnknownMS) - } - if j := phase(w, PhaseJudge, 0); j.State != PhaseMeasured || j.Start == nil || !j.Start.Equal(built) { - t.Fatalf("the judgement is measured from the later moment: %+v", j) - } -} diff --git a/internal/inventory/walksaid.go b/internal/inventory/walksaid.go deleted file mode 100644 index 95c5749a..00000000 --- a/internal/inventory/walksaid.go +++ /dev/null @@ -1,81 +0,0 @@ -package inventory - -import "time" - -// A walk says when it starts and when it ends (novox/hq ADR 0308 decision 1): the save that starts it and the -// save that ends it are marked, so `plan-moved` names what the walk touches, its bound and its outcome on those -// two saves, and on no other. The marks are kept in the walk's times, so a controller replaced mid-walk (a core -// walk replaces it) neither says a start again nor misses the end. - -// The moments of a walk `plan-moved` names. -const ( - WalkStarted = "started" - WalkEnded = "ended" -) - -// The outcomes of an ended walk (ADR 0308 decision 1). -const ( - WalkDelivered = "delivered" - WalkFailed = "failed" - WalkStopped = "stopped" -) - -// Live says the walk is walking: cut, not waiting for its delivery's word, and not ended. -func (p Plan) Live() bool { return p.Open() && !p.Batch() && !p.Waiting() } - -// Ended says the walk is in a state it does not leave by itself: done, failed or superseded. -func (p Plan) Ended() bool { - return p.State == PlanDone || p.State == PlanFailed || p.State == PlanSuperseded -} - -// Outcome is how an ended walk ended: delivered when done; stopped when a person or the delivery's owner stopped -// it, or a newer walk took it over; failed otherwise. Empty for a walk not ended. -func (p Plan) Outcome() string { - switch { - case p.State == PlanDone: - return WalkDelivered - case p.State == PlanSuperseded: - return WalkStopped - case p.State == PlanFailed && p.Delivery != nil && p.Delivery.Stopped != "": - return WalkStopped - case p.State == PlanFailed: - return WalkFailed - } - return "" -} - -// markWalk marks the save that starts or ends the walk, in its times and in WalkSaid, and returns what undoes -// the mark should the save not be written. A walk starts on the first save that finds it live — when it is cut -// and needs no word, or when it is let go — which is before its first tier is asked, and so before anything is -// sent: what it restarts is held from its first send on. It starts again when a failed walk is retried. It ends -// on the first save that finds it ended after it started; one that ends without ever starting (a batch that -// moved nothing, a walk stopped while it waited for its word) touched nothing and says neither. A walk kept -// before its times were (ADR 0282 decision 6) is not marked; a release plan is, its times holding only these. -func (p *Plan) markWalk(now time.Time) (undo func()) { - p.WalkSaid = "" - if p.Batch() || (p.Times == nil && p.Release == nil) { - return func() {} - } - var was *PlanTimes - if p.Times != nil { - kept := *p.Times - was = &kept - } - times := PlanTimes{} - if p.Times != nil { - times = *p.Times - } - at := now.UTC() - switch { - case p.Live() && (times.Started == nil || times.Ended != nil): - times.Started, times.Ended = &at, nil - p.WalkSaid = WalkStarted - case p.Ended() && times.Started != nil && times.Ended == nil: - times.Ended = &at - p.WalkSaid = WalkEnded - default: - return func() {} - } - p.Times = × - return func() { p.Times, p.WalkSaid = was, "" } -} diff --git a/internal/inventory/walksaid_integration_test.go b/internal/inventory/walksaid_integration_test.go deleted file mode 100644 index 6bd21f1f..00000000 --- a/internal/inventory/walksaid_integration_test.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - cut := time.Now().UTC().Add(-time.Minute) - p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding, - Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}}, - Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} - var said []string - was := PlanSaved - PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) } - t.Cleanup(func() { PlanSaved = was }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf { - t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times) - } - // Read back by another controller and saved: not said again. - back.State = PlanRolling - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - // A stale copy ending it is refused, and unmarked. - stale := p - stale.Revision = 1 - stale.State = PlanDone - if err := inv.SavePlan(ctx, &stale); err == nil { - t.Fatal("a stale save is refused") - } - if stale.WalkSaid != "" || stale.Times.Ended != nil { - t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times) - } - back.State = PlanDone - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded { - t.Fatalf("said: %q, want started, nothing, ended", said) - } -} diff --git a/internal/inventory/walksaid_test.go b/internal/inventory/walksaid_test.go deleted file mode 100644 index 6c0a5275..00000000 --- a/internal/inventory/walksaid_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package inventory - -import ( - "testing" - "time" -) - -// A walk is said started on the save that finds it live, ended on the save that finds it ended after it started, -// and neither on any other save (novox/hq ADR 0308 decision 1). -func TestAWalkIsSaidStartedOnceAndEndedOnce(t *testing.T) { - now := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC) - cut := now.Add(-time.Second) - p := Plan{ID: "plan-1", State: PlanBuilding, Tiers: [][]string{{"messenger"}}, - Modules: map[string]*PlanModule{"messenger": {}}, Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} - p.markWalk(now) - if p.WalkSaid != WalkStarted || p.Times.Started == nil || !p.Times.Started.Equal(now) { - t.Fatalf("a live walk's first save starts it: %q %+v", p.WalkSaid, p.Times) - } - p.State = PlanRolling - p.markWalk(now.Add(time.Minute)) - if p.WalkSaid != "" || !p.Times.Started.Equal(now) { - t.Fatalf("a later save says nothing and keeps the start: %q %+v", p.WalkSaid, p.Times) - } - p.State = PlanDone - p.markWalk(now.Add(2 * time.Minute)) - if p.WalkSaid != WalkEnded || p.Times.Ended == nil || p.Outcome() != WalkDelivered { - t.Fatalf("the save that finds it done ends it, delivered: %q %+v %q", p.WalkSaid, p.Times, p.Outcome()) - } - p.markWalk(now.Add(3 * time.Minute)) - if p.WalkSaid != "" { - t.Fatalf("an ended walk saved again says nothing: %q", p.WalkSaid) - } - // Retried, it walks again: a new start. - p.State = PlanBuilding - p.markWalk(now.Add(4 * time.Minute)) - if p.WalkSaid != WalkStarted || p.Times.Ended != nil || !p.Times.Started.Equal(now.Add(4*time.Minute)) { - t.Fatalf("a retried walk starts again: %q %+v", p.WalkSaid, p.Times) - } -} - -// What is never said: a batch, a walk waiting for its word, a walk that ends without having started, and a walk -// kept before its times were. A release plan is said. -func TestWhatIsNotAWalkStartIsNotSaid(t *testing.T) { - now := time.Now() - for name, p := range map[string]Plan{ - "a batch": {State: PlanAssembling, Times: &PlanTimes{}}, - "waiting for a word": {State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery"}}, - "ended, never begun": {State: PlanDone, Times: &PlanTimes{}}, - "kept before times": {State: PlanBuilding}, - "stopped while waits": {State: PlanFailed, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Stopped: "x"}}, - } { - p.markWalk(now) - if p.WalkSaid != "" { - t.Errorf("%s: said %q", name, p.WalkSaid) - } - } - r := Plan{State: PlanRolling, Release: &PlanRelease{Order: []string{"ace"}}} - r.markWalk(now) - if r.WalkSaid != WalkStarted || r.Times == nil || r.Times.Started == nil || r.Times.Class != "" { - t.Fatalf("a release plan is said started, its times holding only that: %q %+v", r.WalkSaid, r.Times) - } - // Let go, the walk starts. - go1 := now - w := Plan{State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Go: &go1}} - w.markWalk(now) - if w.WalkSaid != WalkStarted { - t.Fatalf("a walk let go starts: %q", w.WalkSaid) - } -} - -// An ended walk's outcome: delivered, failed, or stopped by a person or the delivery's owner, or by a newer walk. -func TestAWalksOutcome(t *testing.T) { - for want, p := range map[string]Plan{ - WalkDelivered: {State: PlanDone}, - WalkFailed: {State: PlanFailed}, - WalkStopped: {State: PlanFailed, Delivery: &PlanDelivery{Stopped: "mesh-delivery"}}, - "": {State: PlanRolling}, - } { - if got := p.Outcome(); got != want { - t.Errorf("%s: outcome %q, want %q", p.State, got, want) - } - } - if (Plan{State: PlanSuperseded}).Outcome() != WalkStopped { - t.Error("a superseded walk is stopped") - } -} - -// The marks are kept with the walk, so a controller replaced mid-walk neither says its start again nor misses -// its end; and a save refused leaves the walk unmarked, so the save that is made says it. diff --git a/internal/link/bus.go b/internal/link/bus.go index c4fbdf27..01ba3e7d 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -26,11 +26,6 @@ type Bus interface { // nobody is obliged to act (ADR 0041). PublishEvent(ctx context.Context, key, source, node string, body []byte) error - // PublishDeclaration delivers one node what it should be. Addressed to that node alone: a - // declaration is not an event, and replaying yesterday's is actively harmful - // (design 29 §4, the *state* shape). - PublishDeclaration(ctx context.Context, node string, body []byte) error - // PublishSeatEvent states a fact under a role's own name, for the holder of that role. A // module's event is addressed to the module; a role's is addressed to the role, so it keeps // meaning when the holder changes (novox/hq ADR 0121, ADR 0129). @@ -101,11 +96,6 @@ func SeatEventSubject(seat, event string) string { return "mesh.seat." + seat + ".event." + event } -// DeclareSubject is where one node's declaration lands. Last-per-subject on the NODES stream, so -// a node that was away gets exactly the current one and a replayed older one is refused by -// sequence — the wire-level answer to novox/hq issue 107. -func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" } - func (b OverNATS) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { id, err := eventID() if err != nil { @@ -192,14 +182,6 @@ func (b OverNATS) MaxPayload() int64 { return b.Conn.MaxPayload() } -func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error { - _, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx)) - if err != nil { - return fmt.Errorf("declaring to %s: %w", node, err) - } - return nil -} - // ToolSubject is where a module answers. Derived from the module and the tool, so a caller names // what it wants rather than where it lives. func ToolSubject(module, tool string) string { return "mesh.mod." + module + ".tool." + tool } diff --git a/internal/link/bus_test.go b/internal/link/bus_test.go index 24387a45..4b99616e 100644 --- a/internal/link/bus_test.go +++ b/internal/link/bus_test.go @@ -12,16 +12,6 @@ import ( // // go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus) -// The subject a declaration lands on is one node's, and nothing else's — the state shape. -func TestADeclarationIsAddressedToOneNode(t *testing.T) { - if got := DeclareSubject("anchor"); got != "mesh.node.anchor.declare" { - t.Fatalf("a declaration would go to %q", got) - } - if DeclareSubject("anchor") == DeclareSubject("laptop") { - t.Fatal("two nodes share a declaration subject, so each would apply the other's") - } -} - // A module cannot emit under another's name: the subject is derived from the source, and the // server's permissions make that subject the authority. func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) { diff --git a/internal/link/contracts.go b/internal/link/contracts.go index e139fe6f..20f3a9f2 100644 --- a/internal/link/contracts.go +++ b/internal/link/contracts.go @@ -25,7 +25,7 @@ var Contracts = map[string]Contract{ "report sequence (order.go); an older account is refused and counted. A report from a node-engine " + "that orders nothing is judged by the digest it names (issue 267)", Tests: []string{"TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence", - "TestAnOlderReportOfTheSameDeclarationIsRefused", "TestAnAccountOfAnOlderDeclarationDoesNotReplaceTheNewer", + "TestAnOlderReportOfTheSameDeclarationIsRefused", "TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence", "TestAnAccountIsOlderByEpochThenSequenceThenReportSequence"}}, KindBuilt: {Unordered: "a build's outcome on the retired build seat's subjects: dropped, nothing takes a build in " + "from the bus any more (R4g, ADR 0501 §1); the subjects stay followed while the installer's carried user list " + diff --git a/internal/link/declare.go b/internal/link/declare.go deleted file mode 100644 index be53528b..00000000 --- a/internal/link/declare.go +++ /dev/null @@ -1,94 +0,0 @@ -package link - -import ( - "context" - "encoding/json" - "fmt" - "time" -) - -// Signer is whatever holds the control plane's signing key. -type Signer interface { - Sign(ctx context.Context, message []byte) ([]byte, error) -} - -// ActingGate is what every send passes before it is made (novox/hq to-be 45 §6): whether this process -// may act under the controller's lease. Set by the controller; nil passes every send — a test, a tool. -var ActingGate func(ctx context.Context) error - -// Declare sends a node what it should be, signed. -// -// The signature is over the declaration exactly as it is published — the same bytes the node -// verifies. Anything that re-encoded between here and there would produce a signature over -// something else, and the node would refuse a declaration that was genuinely the mesh's. -// -// Published to the node's own queue, which its account alone may read. -func Declare(ctx context.Context, bus Bus, signer Signer, node string, - declaration []byte, timeout time.Duration) error { - - if !json.Valid(declaration) { - return fmt.Errorf("refusing to send %s something that is not a declaration", node) - } - // **At the send, not only where it was composed**: a lease lost between the two stops this one. - if ActingGate != nil { - if err := ActingGate(ctx); err != nil { - return fmt.Errorf("%s was not sent its declaration: %w", node, err) - } - } - - signature, err := signer.Sign(ctx, declaration) - if err != nil { - return fmt.Errorf("cannot sign a declaration for %s: %w", node, err) - } - - body, err := Envelope(declaration, signature) - if err != nil { - return err - } - - // **Measured before it is sent** (novox/hq issue 1689): the bare bus error said only "maximum payload - // exceeded", on 2026-10-21 for the control node's declaration, 2 KB over. The limit is the connection's own. - if limit := payloadLimitOf(bus); limit > 0 { - size := int64(len(body)) - if DeclarationMeasured != nil { - DeclarationMeasured(ctx, node, size, limit) - } - if size > limit { - return fmt.Errorf("%s's declaration is %d bytes on the wire (%d bytes of declaration, signed and encoded), "+ - "over the bus's largest message of %d bytes by %d: it cannot be sent (novox/hq issue 1689)", - node, size, len(declaration), limit, size-limit) - } - } - - publish, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - return bus.PublishDeclaration(publish, node, body) -} - -// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON -// document. Its length is what the bus's largest message bounds. -func Envelope(declaration, signature []byte) ([]byte, error) { - return json.Marshal(Signed{Declaration: declaration, Signature: signature}) -} - -// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that -// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486 -// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline). -const NearTheLimit = 0.75 - -// DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message, -// so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689). -// Set by the controller; nil in a test or a tool. -var DeclarationMeasured func(ctx context.Context, node string, size, limit int64) - -// payloadLimiter is a bus that knows its largest message. -type payloadLimiter interface{ MaxPayload() int64 } - -// payloadLimitOf is the bus's largest message in bytes, or 0 when the bus does not say. -func payloadLimitOf(bus Bus) int64 { - if l, ok := bus.(payloadLimiter); ok { - return l.MaxPayload() - } - return 0 -} diff --git a/internal/link/declare_room_integration_test.go b/internal/link/declare_room_integration_test.go deleted file mode 100644 index eb607b2f..00000000 --- a/internal/link/declare_room_integration_test.go +++ /dev/null @@ -1,52 +0,0 @@ -//go:build integration - -package link - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/testbus" -) - -// Against a real bus whose largest message is small: the limit is read from the connection, the one that fits goes -// through, the one over is refused by us with its size and the limit, not by the client's bare error (issue 1689). -func TestADeclarationIsMeasuredAgainstTheRealBusLimit(t *testing.T) { - s := testbus.StartWithMaxPayload(t, 2048) - conn, err := nats.Connect(s.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - js, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := js.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}}); err != nil { - t.Fatal(err) - } - bus := OverNATS{Conn: conn, JS: js} - if bus.MaxPayload() != 2048 { - t.Fatalf("the limit read from the connection is %d, expected 2048", bus.MaxPayload()) - } - var told []int64 - was := DeclarationMeasured - DeclarationMeasured = func(_ context.Context, _ string, size, limit int64) { told = append(told, size, limit) } - t.Cleanup(func() { DeclarationMeasured = was }) - - fits := []byte(`{"a":"` + strings.Repeat("x", 600) + `"}`) - if err := Declare(t.Context(), bus, fixedSigner{}, "novox", fits, 5*time.Second); err != nil { - t.Fatalf("a declaration under the limit: %v", err) - } - over := []byte(`{"a":"` + strings.Repeat("x", 1600) + `"}`) - err = Declare(t.Context(), bus, fixedSigner{}, "novox", over, 5*time.Second) - if err == nil || !strings.Contains(err.Error(), "2048 bytes") || strings.Contains(err.Error(), "maximum payload exceeded") { - t.Fatalf("a declaration over the limit: %v", err) - } - if len(told) != 4 || told[1] != 2048 || told[2] <= 2048 { - t.Fatalf("measured %v", told) - } -} diff --git a/internal/link/declare_room_test.go b/internal/link/declare_room_test.go deleted file mode 100644 index 4339277b..00000000 --- a/internal/link/declare_room_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package link - -import ( - "context" - "strings" - "testing" - "time" -) - -type roomBus struct { - Bus - limit int64 - sent [][]byte -} - -func (b *roomBus) MaxPayload() int64 { return b.limit } -func (b *roomBus) PublishDeclaration(_ context.Context, _ string, body []byte) error { - b.sent = append(b.sent, body) - return nil -} - -type fixedSigner struct{} - -func (fixedSigner) Sign(context.Context, []byte) ([]byte, error) { return []byte("sig"), nil } - -// A declaration is measured as it goes on the wire against the bus's own limit; one over it is refused saying size and -// limit, and what the measure sees is told to whoever raises a condition (issue 1689). -func TestADeclarationOverTheBusLimitIsRefusedSayingSizeAndLimit(t *testing.T) { - var seen []int64 - was := DeclarationMeasured - DeclarationMeasured = func(_ context.Context, node string, size, limit int64) { - if node != "novox" || limit != 400 { - t.Errorf("measured %s against %d", node, limit) - } - seen = append(seen, size) - } - t.Cleanup(func() { DeclarationMeasured = was }) - - bus := &roomBus{limit: 400} - small := []byte(`{"a":"` + strings.Repeat("x", 100) + `"}`) - if err := Declare(t.Context(), bus, fixedSigner{}, "novox", small, time.Second); err != nil || len(bus.sent) != 1 { - t.Fatalf("a small declaration: %v, %d sent", err, len(bus.sent)) - } - if int64(len(bus.sent[0])) != seen[0] { - t.Fatalf("measured %d bytes, sent %d", seen[0], len(bus.sent[0])) - } - - big := []byte(`{"a":"` + strings.Repeat("x", 300) + `"}`) - err := Declare(t.Context(), bus, fixedSigner{}, "novox", big, time.Second) - if err == nil || !strings.Contains(err.Error(), "400 bytes") || !strings.Contains(err.Error(), "novox") || - !strings.Contains(err.Error(), "over") || strings.Contains(err.Error(), "maximum payload exceeded") { - t.Fatalf("an oversize declaration: %v", err) - } - if len(bus.sent) != 1 || len(seen) != 2 || seen[1] <= 400 { - t.Fatalf("it was sent (%d) or not measured (%v)", len(bus.sent), seen) - } -} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 845d9c01..24e4b085 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -11,7 +11,6 @@ import ( "fmt" "log" "sort" - "strings" "time" "github.com/novox/mesh-controller/internal/broker" @@ -268,16 +267,6 @@ var _ Enroller = Enrolment{} // A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is // its own account of its own machine, kept as a copy for recovery — so this writes it down and // decides nothing from it. -// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not -// acted on (design 25 §3, window.go). -// -// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great -// many things that record reports have no idea what was sent — every test in this package among -// them. So the serving loop asks for this when the listener happens to be able to answer, and -// where it cannot, a report has nothing to be stale against and is simply acted on. -func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) { - return e.Inventory.Outstanding(ctx, node) -} func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err error) { // A store that could not be asked right now is said as such, so the report is kept for @@ -444,22 +433,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err // arrives; the digest the mesh last sent decides nothing for it. **By digest, from an older // node-engine**: its report claims no order, and the rule of issue 267 stands for it. // - // And whether the node-engine reads an epoch in a declaration, from every account it gives: the - // mesh sends one only to a machine that said so, and one rolled back says so no longer. - if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil { - return false, err - } - // And whether it reads a generation (novox/hq issue 234), the same way: only from a node-engine that - // orders its reports, so a one-shot report (a rekey) does not say an engine stopped reading one. - if report.Ordered() { - if err := e.Inventory.RecordReadsGeneration(ctx, node.ID, report.ReadsGeneration); err != nil { - return false, err - } - // And whether it reads a module per resource and a build per module (novox/hq issue 1145), the same way. - if err := e.Inventory.RecordReadsModules(ctx, node.ID, report.ReadsModules); err != nil { - return false, err - } - } if report.Ordered() { account := AccountOf(report) news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing, @@ -478,17 +451,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err return false, err } } else { - if report.Declared != "" { - sent, err := e.Inventory.Outstanding(ctx, report.Node) - if err != nil { - return false, err - } - if Superseded(report.Declared, sent) { - log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+ - "the mesh has sent it %s since", report.Node, short(report.Declared), short(sent)) - return false, e.Inventory.Seen(ctx, node.ID) - } - } // **Whether this is news** is the store's answer: it holds the previous report, and a machine // that reconciles every minute says the same thing until something changes (novox/hq ADR 0134). news, err = e.Inventory.RecordDoing(ctx, node.ID, doing) @@ -496,27 +458,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err return false, err } } - // And how long the machine took, from the send to this first account of it (novox/hq to-be 45 - // Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure. - // - // **A wait for a busy container is its own measure** (novox/hq ADR 0335): the first report that says one - // waits is the machine's answer, kept as its apply with the detail ApplyBusy; the report that says the - // container replaced ends the wait, kept as a `busy-wait` of the same send — so the apply's time holds no - // busy time, and the walk reads the machine as answered and waiting, never as silent. - outcome := doing.Outcome - if doing.Waiting() { - outcome = inventory.ApplyBusy - log.Printf("%s applied declaration %s and leaves a busy container running until it says idle: %s", - report.Node, short(report.Declared), busyWords(doing.Busy)) - } else if replacedAfterWaiting(doing.Busy) { - if err := e.Inventory.RecordBusyDuration(ctx, report.Node, report.Declared); err != nil { - log.Printf("how long %s waited for a busy container could not be recorded: %v", report.Node, err) - } - } - if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, outcome); err != nil { - log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err) - } - // A refusal, a failure, or a bare word that the node is there — none of them is an account of // what the machine holds, so each moves last_seen and nothing else. Recording a partial list // as though it were the whole would tell a rebuilding node to remove what it still has. @@ -525,25 +466,3 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err } return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied) } - -// busyWords is every container waiting because it said busy, in a line (novox/hq ADR 0335). -func busyWords(busy []inventory.BusyWait) string { - var out []string - for _, b := range busy { - if b.State == inventory.BusyWaiting { - out = append(out, fmt.Sprintf("%s since %s, at the latest until %s", b.Container, - b.Since.UTC().Format(time.RFC3339), b.Until.UTC().Format(time.RFC3339))) - } - } - return strings.Join(out, "; ") -} - -// replacedAfterWaiting says a report replaced a container that had waited because it was busy. -func replacedAfterWaiting(busy []inventory.BusyWait) bool { - for _, b := range busy { - if b.State != inventory.BusyWaiting { - return true - } - } - return false -} diff --git a/internal/link/events.go b/internal/link/events.go index 551a9039..1ef79499 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -73,15 +73,7 @@ const ( // KeyRolledBack: a build failed its gate on its first machine and was put back there, or could not // be (novox/hq ADR 0236, to-be 45 §8); or a witness on a machine put a core component back. KeyRolledBack = "rolled-back" - // KeySent: a plan's tier was sent to a machine, and the send kept (novox/hq ADR 0406 §5, issue 974): the plan, the tier, - // the machine, the modules and when. What an outsider hears of a rollout; the controller's own next step does not - // wait for it. - KeySent = "sent" - // KeyPlanMoved: a walk — the controller's plan of one trunk commit, tier by tier across the machines — - // was kept in a new state (novox/hq ADR 0239): the plan whole, for the delivery it walks to read its - // steps from. Said after every save of the serving controller; a delivery's holder also asks for the - // walks it follows, so a save made elsewhere is found by comparison, never lost. - KeyPlanMoved = "plan-moved" + // (No plan is kept or sent any more, novox/hq ADR 0539: the events plan-moved and sent are gone with them.) // KeyAssignmentChanged and KeyDeclaredConfigChanged: a key of the controller's assignments or declared-config // mirror was written or removed (novox/hq ADR 0289, design 54 §3, design 57 F3): which key, how and at which // revision of the bucket — never the value, which the bucket holds. diff --git a/internal/link/generation_test.go b/internal/link/generation_test.go deleted file mode 100644 index b93e41f4..00000000 --- a/internal/link/generation_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package link - -import ( - "encoding/json" - "testing" -) - -// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it: -// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal -// of a declaration composed from an older generation than the machine applied. -func TestTheGenerationOnTheWire(t *testing.T) { - raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` + - `"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`) - var r Report - if err := json.Unmarshal(raw, &r); err != nil { - t.Fatal(err) - } - if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) || - r.Sequence != 12 { - t.Fatalf("a node-engine's generation refusal reads as %+v", r) - } - // Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender. - if r.StaleRefusalOf() { - t.Errorf("a generation refusal reads as a stale writer's") - } - var older Report - if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil { - t.Fatal(err) - } - if older.ReadsGeneration || older.OlderGeneration != nil { - t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older) - } -} - -// 1145/AC3: a report's word that the node-engine reads a module per resource and a build per module, as mesh-host's -// report writes it: `reads_modules`, on every report of a node-engine that reads them. -func TestWhetherTheNodeEngineReadsModulesIsOnItsReport(t *testing.T) { - var reads, older Report - if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":8,"reads_modules":true}`), &reads); err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil { - t.Fatal(err) - } - if !reads.ReadsModules || older.ReadsModules { - t.Fatalf("1145/AC3: reads %v, an older node-engine reads %v", reads.ReadsModules, older.ReadsModules) - } -} diff --git a/internal/link/heard_order_integration_test.go b/internal/link/heard_order_integration_test.go index 5941b196..35ba28e5 100644 --- a/internal/link/heard_order_integration_test.go +++ b/internal/link/heard_order_integration_test.go @@ -70,32 +70,22 @@ func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) { } func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) { - inv, heard, id := anOrderedMachine(t) + inv, heard, _ := anOrderedMachine(t) ctx := context.Background() - if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { - t.Fatal(err) - } if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { t.Fatal(err) } if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" { t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing) } - // And the machine reads an epoch: it orders its reports. - if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads { - t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err) - } } -func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) { +func TestAnUnorderedAccountClearsTheOrder(t *testing.T) { inv, heard, id := anOrderedMachine(t) ctx := context.Background() if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { t.Fatal(err) } - if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { - t.Fatal(err) - } if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil { t.Fatal(err) } @@ -105,9 +95,6 @@ func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) { if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) { t.Fatalf("an unordered account left the order kept: %+v", kept) } - if reads, _ := inv.ReadsEpoch(ctx, id); reads { - t.Fatal("a node-engine that orders nothing is still sent an epoch") - } // The next ordered account, whatever its numbers, has nothing to be older than. if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil { t.Fatal(err) diff --git a/internal/link/heard_order_test.go b/internal/link/heard_order_test.go index de59da59..3ef40105 100644 --- a/internal/link/heard_order_test.go +++ b/internal/link/heard_order_test.go @@ -1,7 +1,6 @@ package link_test import ( - "testing" "time" "github.com/novox/mesh-controller/internal/link" @@ -25,27 +24,6 @@ func ordered(declared string, epoch, sequence, report int64, applied ...string) // A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the // order kept is cleared, and it is no longer sent an epoch. -// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose -// own report was lost is still counted from the machine's own tally. -func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) { - count := link.NewRefusalCount() - now := time.Now() - refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11}, - OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3} - if !refusal.StaleRefusalOf() { - t.Fatal("a refusal naming the order it holds is not stale") - } - count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it - count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now}) - count.Lifetime("anchor", refusal.RefusedOlder, now) - count.Lifetime("anchor", 5, now) // two more refused, their reports lost - got := count.Within(now.Add(-time.Minute)) - if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" || - got[1].Count != 1 || got[1].Receivers[0] != "anchor" { - t.Fatalf("the refusals by writer are %+v", got) - } -} - // countFor is how many refusals of a writer this process has heard in the last minute. func countFor(writer string) int { for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) { diff --git a/internal/link/heard_test.go b/internal/link/heard_test.go index 93a2dd0d..05d0e6f1 100644 --- a/internal/link/heard_test.go +++ b/internal/link/heard_test.go @@ -88,7 +88,7 @@ func TestACleanApplyIsRecordedAsOne(t *testing.T) { } } -func TestABareAliveDoesNotWipeTheDeclarationThatSaysANodeIsCurrent(t *testing.T) { +func TestABareAliveDoesNotWipeTheLastRealApply(t *testing.T) { // A node reports it is alive every minute and describes what it applied rarely. If a bare // alive were written down as a report it would replace the last real apply with an empty one // — clearing the declaration digest current is measured against — so a node that had just @@ -102,14 +102,13 @@ func TestABareAliveDoesNotWipeTheDeclarationThatSaysANodeIsCurrent(t *testing.T) } // The mesh sent this node a declaration, and the node applied it and named which by digest. const digest = "d640d1b6a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f6071829304152" - if err := inv.RecordSent(ctx, node.ID, digest, nil); err != nil { - t.Fatal(err) - } if _, err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{ Node: "anchor", Applied: []string{"a", "b"}, Declared: digest, Carried: []int{5432}, }); err != nil { t.Fatal(err) } + // Whether the last real apply is still there: its outcome and the declaration it was about. Whether a node is current is + // read from its haves (cmd/mesh-controller lastReports), not from a report. currentOf := func(name string) bool { reports, err := inv.LastReports(ctx) if err != nil { @@ -117,14 +116,14 @@ func TestABareAliveDoesNotWipeTheDeclarationThatSaysANodeIsCurrent(t *testing.T) } for _, r := range reports { if r.Node == name { - return r.Current + return r.Outcome == inventory.OutcomeApplied && r.Declared == digest } } t.Fatalf("no report for %s", name) return false } if !currentOf("anchor") { - t.Fatal("a node that applied exactly what it was sent does not read as current") + t.Fatal("the last real apply is not kept") } // Now the node says only that it is there, as it does every minute. @@ -263,48 +262,12 @@ func TestWhatFiltersAMachineIsKeptFromItsReport(t *testing.T) { } } -// Measured on the home server (novox/hq issue 267): the mesh sent d2; the machine applied it and -// reported, and a reconcile's report about d1 — made just before d2 arrived — reached the mesh after -// it. Stored last, it read as the machine never having applied d2, and the release plan waited on a -// report it had already been given. -func TestAnAccountOfAnOlderDeclarationDoesNotReplaceTheNewer(t *testing.T) { - inv := inventory.ForTest(t) - ctx := context.Background() - node, err := inv.AddNode(ctx, "home-server") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil { - t.Fatal(err) - } - heard := link.Enrolment{Inventory: inv} - if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2", - Applied: []string{"a", "b"}, Outward: []string{"eth0"}}); err != nil { - t.Fatal(err) - } - if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1", - Applied: []string{"a"}, Outward: []string{"eth1"}}); err != nil { - t.Fatal(err) - } - doing, said, err := inv.DoingOf(ctx, "home-server") - if err != nil || !said { - t.Fatalf("no account kept: %v", err) - } - if doing.Declared != "d2" || doing.Applied != 2 { - t.Fatalf("the older report replaced the account of what was sent: %+v", doing) - } - // What it says about the machine is kept whenever it arrives, as before. - if links, err := inv.OutwardLinksOf(ctx, "home-server"); err != nil || len(links) != 1 || links[0] != "eth1" { - t.Fatalf("what the older report said about the machine was not kept: %v %v", links, err) - } -} - // The account of the declaration that is outstanding replaces whatever was kept, and so does one // from a machine nothing was ever recorded as sent to. func TestAnAccountOfTheSentDeclarationReplacesTheKeptOne(t *testing.T) { inv := inventory.ForTest(t) ctx := context.Background() - node, err := inv.AddNode(ctx, "home-server") + _, err := inv.AddNode(ctx, "home-server") if err != nil { t.Fatal(err) } @@ -312,9 +275,6 @@ func TestAnAccountOfTheSentDeclarationReplacesTheKeptOne(t *testing.T) { if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1", Applied: []string{"a"}}); err != nil { t.Fatal(err) } - if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil { - t.Fatal(err) - } if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2", Applied: []string{"a", "b"}}); err != nil { t.Fatal(err) } diff --git a/internal/link/order.go b/internal/link/order.go index 311bd0fd..ed8de121 100644 --- a/internal/link/order.go +++ b/internal/link/order.go @@ -2,68 +2,22 @@ package link import "strconv" -// The order a declaration carries, and the order a report about one carries back (novox/hq to-be 45 -// §6, ADR 0227 rule 2). +// The order of a node's facts (novox/hq ADR 0539 §9, to-be 45 §6, ADR 0227 rule 2). // -// **This file is the controller's side of the contract with the node-engine for Phase 2**, and the one -// place it is written here. It mirrors mesh-host internal/link/messages.go (`Order`, and the report's -// `report_sequence`, `older_than`, `refused_older`) field for field and rule for rule, as Report has -// always mirrored the host's report: a key added on one side and not the other is a key the other side -// does not know exists. -// -// **The wire.** -// -// - A declaration's order is two top-level keys of its signed envelope, beside "declaration" and -// "resources": `sequence` (since issue 107) and `epoch` (new). Inside the signed bytes, so a message -// cannot be given a newer order than the controller gave it. -// - A report carries the order of the declaration it is about as the same two top-level keys, beside -// "declared"; `report_sequence`, the node-engine's own number for the report; `older_than`, on a -// refusal of a declaration older than one it applied, the order of the one it holds; and -// `refused_older`, how many it has refused so, ever, on every report. -// -// Every key is optional and absent when zero, and zero is "no order claimed", never "first". So each -// side reads the other's older shape: -// -// - **An older node-engine with this controller.** It decodes a declaration strictly and refuses a key -// it does not know, whole — so `epoch` is sent only to a machine whose latest report carried a -// `report_sequence`, which a node-engine that reads the epoch always does. Until then it is sent the -// sequence alone, as today; its reports carry no report sequence and are judged by the digest they -// name, as today (issue 267). -// - **A newer node-engine with an older controller.** It is sent no epoch, which claims none: it -// refuses nothing by epoch. The keys it adds to a report are fields the older controller ignores. -// - **A controller rolled back to a build without the lease** sends no epoch again and is not refused -// for it: a node-engine refuses by epoch only when both declarations claim one. That gives up the -// epoch's protection while such a build runs — the price of a rollback that cannot strand every -// machine. -// -// **Epoch** is the controller lease's epoch (to-be 45 §6): the lease bucket's revision at which the -// instance that composed the declaration took the lease. It only grows: a later holder's is higher, and -// the controller never issues one at or under the highest it has issued (internal/lease, the floor). -// **Sequence** is the declaration's number for that machine, one higher every send, taken from the -// controller's store before the declaration is composed (issues 107, 204). +// A node-engine says its facts (profile, firewall found, reachable, tunnel, witness, rollbacks, rekey, health) in a report +// on the bus, and orders them itself: `epoch` and `sequence` are the node-engine's own (Writer `node-engine`), and +// `report_sequence` numbers each report it makes. The controller keeps the newest account of a machine and refuses an older +// one, said and counted (rule 2): by epoch, then sequence, then report sequence, each compared only where both claim one. +// Zero is "no order claimed", never "first". The controller sends a machine no declaration to be ordered by: the order here +// is only ever the machine's own. -// Order is where a declaration stands among everything the mesh has sent a machine: the lease epoch it -// was sent under and its sequence. Zero in either claims none. +// Order is where a node-engine's account stands among its own: the epoch and sequence the node-engine counts. Zero in either +// claims none. type Order struct { Epoch int64 `json:"epoch,omitempty"` Sequence int64 `json:"sequence,omitempty"` } -// Older is the node-engine's rule, as mesh-host states it: a declaration of this order is older than -// one of order `than` the machine applied **only when both claim an epoch** — and then when its epoch -// is lower, or its epoch is the same and its sequence lower (both claimed). A higher epoch is a new lease -// holder and is never older, whatever its sequence. Here so the controller's tests state what the -// machines will do with what it sends. -func (o Order) Older(than Order) bool { - if o.Epoch <= 0 || than.Epoch <= 0 { - return false - } - if o.Epoch != than.Epoch { - return o.Epoch < than.Epoch - } - return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence -} - // Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a Stringer // promoted onto every report would print each one as its order alone. func (o Order) Words() string { @@ -91,17 +45,6 @@ func AccountOf(r Report) Account { return Account{Order: r.Order, ReportSequence // before (issue 267). func (r Report) Ordered() bool { return r.ReportSequence > 0 } -// ReadsEpoch says the node-engine that made the report takes an epoch in a declaration: one that orders -// its reports does (mesh-host: "its presence also says this host reads a declaration's epoch"). -func (r Report) ReadsEpoch() bool { return r.ReportSequence > 0 } - -// StaleRefusalOf says a report refuses a declaration older than one the machine applied: the node-engine -// names the order it holds (`older_than`), or, from a node-engine older than that, says so in the words -// of its sequence refusal. -func (r Report) StaleRefusalOf() bool { - return r.OlderThan != nil || (r.Refused != "" && IsStaleRefusal(r.Refused)) -} - // OlderThan is the controller's rule (to-be 45 §6): whether this account is older than the one kept for // the machine — **by epoch, then sequence, then report sequence**, each compared only where both claim // one. An account of a declaration of an older epoch is refused; of an older sequence, refused; an older diff --git a/internal/link/order_test.go b/internal/link/order_test.go index a215ba08..d0a11101 100644 --- a/internal/link/order_test.go +++ b/internal/link/order_test.go @@ -5,34 +5,8 @@ import ( "testing" ) -// The contract of order.go, case by case: what a node-engine refuses of a declaration (the rule -// mesh-host states, restated so the controller's tests say what the machines do with what it sends), -// what the controller refuses of a report, and the bytes on the wire both ways. - -func TestADeclarationIsOlderOnlyWhenBothClaimAnEpoch(t *testing.T) { - for _, c := range []struct { - name string - arriving Order - held Order - older bool - }{ - {"a newer sequence of the same epoch", Order{57, 12}, Order{57, 11}, false}, - {"the same declaration again (a reconcile)", Order{57, 12}, Order{57, 12}, false}, - {"a lower sequence of the same epoch", Order{57, 11}, Order{57, 12}, true}, - // Issue 204: a controller that lost its lease goes on sending. - {"an older epoch, whatever its sequence", Order{41, 99}, Order{57, 12}, true}, - {"a newer epoch, whatever its sequence", Order{57, 3}, Order{41, 99}, false}, - {"no epoch arriving: a rolled-back controller is not stranded", Order{0, 11}, Order{57, 12}, false}, - {"no epoch held: what the machine held before any lease", Order{57, 11}, Order{0, 12}, false}, - {"no order at all", Order{}, Order{57, 12}, false}, - } { - t.Run(c.name, func(t *testing.T) { - if got := c.arriving.Older(c.held); got != c.older { - t.Fatalf("%s against %s: older %v, want %v", c.arriving.Words(), c.held.Words(), got, c.older) - } - }) - } -} +// The contract of order.go, case by case: what the controller refuses of a node-engine's facts (the node-engine's own order) +// and the bytes on the wire. func TestAnAccountIsOlderByEpochThenSequenceThenReportSequence(t *testing.T) { for _, c := range []struct { @@ -78,7 +52,7 @@ func TestTheOrderOnTheWire(t *testing.T) { r.RefusedOlder != 3 { t.Fatalf("a node-engine's stale refusal reads as %+v", r) } - if !r.Ordered() || !r.ReadsEpoch() || !r.StaleRefusalOf() { + if !r.Ordered() { t.Fatalf("a node-engine that orders its reports reads as one that does not: %+v", r) } @@ -87,15 +61,9 @@ func TestTheOrderOnTheWire(t *testing.T) { if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"declared":"d1"}`), &older); err != nil { t.Fatal(err) } - if older.Ordered() || older.ReadsEpoch() || older.StaleRefusalOf() { + if older.Ordered() { t.Fatalf("an older node-engine's report reads as ordered: %+v", older) } - // Its stale refusal, in the words it has always used, is still one. - older.Refused = "this declaration " + StaleRefusal + ": it is sequence 3" - if !older.StaleRefusalOf() { - t.Fatal("an older node-engine's refusal of an older sequence does not read as stale") - } - // And a report with no order puts no order key on the wire. raw, err := json.Marshal(Report{Node: "anchor", Declared: "d1"}) if err != nil { diff --git a/internal/link/receive.go b/internal/link/receive.go index 30e053cf..aeb12417 100644 --- a/internal/link/receive.go +++ b/internal/link/receive.go @@ -121,15 +121,3 @@ type Inbound interface { // Close lets go of whatever the implementation holds. Close() } - -// Outstanding answers which declaration the mesh last sent a node — the digest, not the -// declaration. -// -// **Asked before the store is waited on** (design 25 §3): a report about a declaration the mesh -// has already moved past is not worth holding a slot in the window that a current message needs. -// It is a separate interface from Listener rather than a method on it, because a controller that -// only publishes needs neither and something that records reports need not also be able to say -// what was sent. -type Outstanding interface { - Outstanding(ctx context.Context, node string) (string, error) -} diff --git a/internal/link/receive_nats_integration_test.go b/internal/link/receive_nats_integration_test.go index 59c89e1d..b2ee6f5a 100644 --- a/internal/link/receive_nats_integration_test.go +++ b/internal/link/receive_nats_integration_test.go @@ -102,25 +102,6 @@ func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T }) } -func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { - js := aBus(t) - store := &sentAndHeardSafely{sent: "d2"} - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "the superseded report leaving the stream", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) - if store.count() != 0 { - t.Fatalf("a report about a superseded declaration was acted on") - } -} - func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { js := aBus(t) s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, diff --git a/internal/link/serve.go b/internal/link/serve.go index 4aeff2dd..3e0ccf42 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -292,26 +292,6 @@ func (s *Server) giveUpAfter() time.Duration { return s.giveUp } -// outstanding is the digest of the declaration the mesh last sent a node. -// -// Nothing is guessed when it cannot be answered: a listener that keeps no record of what was sent -// gives a report nothing to be stale against, and a store that cannot be read will hold the report -// anyway, so there is nothing for the check to decide. -func (s *Server) outstanding(ctx context.Context, node string) string { - if node == "" { - return "" - } - asks, ok := s.listener.(Outstanding) - if !ok { - return "" - } - digest, err := asks.Outstanding(ctx, node) - if err != nil { - return "" - } - return digest -} - // toolsHeartbeat records that a machine's node tools were heard from (novox/hq to-be 45 S11), and // nothing else: in this process's memory, for the watchdog — the next one is a minute away. func (s *Server) toolsHeartbeat(m Control) { @@ -343,25 +323,8 @@ func (s *Server) reported(ctx context.Context, m Control) { what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared) if s.listener != nil { - // **Asked before the store, not after** (design 25 §3). A report about a declaration the - // mesh has moved past would otherwise wait out a restarting store to be written and then - // overwrite what the node is doing now — and on the bus being built, where the holding is - // the server's, it comes back after the newer was applied whatever the controller does. - outstanding := s.outstanding(ctx, report.Node) - declaredIn := staleAgainst(report) - if report.Ordered() { - // Kept by its order, not by the digest the mesh last sent (novox/hq to-be 45 §6): the - // listener refuses an older account under the one rule, whatever it is about. - declaredIn = "" - } - if Superseded(declaredIn, outstanding) { - s.log.Printf("set aside %s: the mesh has moved past that declaration", what) - _ = m.Took() - return - } - news, err := s.listener.Heard(context.Background(), report) - switch s.decide(ctx, m, what, declaredIn, outstanding, err) { + switch s.decide(ctx, m, what, "", "", err) { case Hold: // Held, not settled, while the store cannot take it: the node reports an apply once, // and a report lost here is a node the mesh never hears from again — the store @@ -396,27 +359,6 @@ func (s *Server) reported(ctx context.Context, m Control) { _ = m.Took() } -// staleAgainst is the declaration a report may be judged stale against, and it is empty for a -// report that is not only an account of an apply. -// -// **A report carries two different things, and only one of them is about a declaration.** What the -// node applied is; what the machine *is* — the tunnel it took over, the ports its own bundle holds, -// what an adopted node found and is keeping, a node moving its overlay key — is not. Those reach -// the mesh on a report because a report is the message a node sends, and nowhere else: a rekey -// dropped as stale is a node whose overlay key never moves, and no retry is coming, because the -// node said it once. -// -// So staleness is asked only of a report that is purely an apply's account. The rest is acted on -// whenever it arrives, which is the behaviour the mesh has had all along. -func staleAgainst(report Report) string { - if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 || - report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 || - len(report.Outward) > 0 { - return "" - } - return report.Declared -} - func (s *Server) enrolling(ctx context.Context, m Control) { reply := EnrolReply{Refusal: "that token cannot be used"} diff --git a/internal/link/signer.go b/internal/link/signer.go new file mode 100644 index 00000000..f083333c --- /dev/null +++ b/internal/link/signer.go @@ -0,0 +1,12 @@ +package link + +import "context" + +// Signer is whatever holds the control plane's signing key. +type Signer interface { + Sign(ctx context.Context, message []byte) ([]byte, error) +} + +// ActingGate is what every act passes before it is made (novox/hq to-be 45 §6): whether this process may act under the +// controller's lease. Set by the controller; nil passes every act — a test, a tool. +var ActingGate func(ctx context.Context) error diff --git a/internal/link/stale_report_test.go b/internal/link/stale_report_test.go deleted file mode 100644 index cf9f303b..00000000 --- a/internal/link/stale_report_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package link - -import ( - "context" - "errors" - "testing" -) - -// Supersession as a check rather than a memory (design 25 §3). -// -// Holding a message in memory let the controller drop an older report when a newer one for the -// same node arrived. On the bus being built the message belongs to the server and comes back -// whatever happened meanwhile — so the older report is redelivered *after* the newer was applied, -// and acting on it would undo the newer. -// -// The answer was already in the message: a report carries the digest of the declaration it is -// about, so "is this the past?" is a question the message answers. - -// sentAndHeard records reports and knows what was last sent, which is the pair the check needs. -type sentAndHeard struct { - sent string - heard []Report - err error -} - -func (s *sentAndHeard) Heard(_ context.Context, r Report) (bool, error) { - if s.err != nil { - return false, s.err - } - s.heard = append(s.heard, r) - return true, nil -} - -func (s *sentAndHeard) Outstanding(context.Context, string) (string, error) { return s.sent, nil } - -// A report about a declaration the mesh has moved past is settled and not acted on. Settled rather -// than dropped, because there is nothing wrong with the message — it is simply the past, and -// redelivering it for ever is worse than letting it go. -func TestAReportAboutASupersededDeclarationIsNotActedOn(t *testing.T) { - store := &sentAndHeard{sent: "d2"} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) - - if len(store.heard) != 0 { - t.Fatalf("a report about a superseded declaration was acted on: %+v", store.heard) - } - if !to.acked { - t.Fatalf("a superseded report was not settled, so it comes back for ever: %+v", *to) - } -} - -// The report about the declaration that *is* outstanding is acted on, and so is one from a node -// the mesh has no digest for — an older host that says nothing about which declaration it applied -// has nothing to be judged against, and refusing it would silence every node built before reports -// carried the digest. -func TestAReportAboutTheOutstandingDeclarationIsActedOn(t *testing.T) { - for _, c := range []struct{ what, sent, declared string }{ - {"the one outstanding", "d2", "d2"}, - {"a report that says nothing about which", "d2", ""}, - {"a node nothing was ever sent", "", "d1"}, - } { - store := &sentAndHeard{sent: c.sent} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", c.declared))) - if len(store.heard) != 1 || !to.acked { - t.Errorf("%s: was not acted on and acknowledged: heard %+v, settled %+v", - c.what, store.heard, *to) - } - } -} - -// **Staleness is decided before the store is waited on**, not after: a redelivery that lost its -// race is not worth holding a slot in the window that a current message needs. -func TestASupersededReportIsNotHeldForTheStore(t *testing.T) { - store := &sentAndHeard{sent: "d2", err: errors.Join(ErrTryAgain, errors.New("starting up"))} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) - if !to.acked || len(in.held) != 0 { - t.Fatalf("a superseded report waited for the store: %+v, %d held", *to, len(in.held)) - } -} - -// **Half of a report is not about a declaration, and that half is never stale.** -// -// What the machine *is* — the tunnel it took over, the ports its own bundle holds, what an adopted -// node found and is keeping, a node moving its overlay key — reaches the mesh on a report and -// nowhere else. A rekey set aside as stale is a node whose overlay key never moves, and no retry is -// coming, because the node said it once. So a report carrying any of these is acted on whenever it -// arrives, however far the mesh has moved on. -func TestAReportCarryingWhatOnlyTheNodeKnowsIsActedOnHoweverOldItIs(t *testing.T) { - for _, c := range []struct { - what string - report Report - }{ - {"a rekey", Report{Node: "anchor", Declared: "d1", - Rekey: &Rekey{Previous: "k1", OverlayKey: "k2"}}}, - {"the tunnel it carried", Report{Node: "anchor", Declared: "d1", - Tunnel: &CarriedTunnel{Interface: "wg0", State: "taken"}}}, - {"what an adopted node holds", Report{Node: "anchor", Declared: "d1", - Held: []Held{{ID: "conf", Module: "web", Kind: "file"}}}}, - {"the firewall it found", Report{Node: "anchor", Declared: "d1", Firewall: "ufw"}}, - {"what is reachable on it", Report{Node: "anchor", Declared: "d1", - Reachable: []Reach{{Protocol: "tcp", Port: 443}}}}, - {"the ports its own bundle holds", Report{Node: "anchor", Declared: "d1", - Carried: []int{5432}}}, - } { - store := &sentAndHeard{sent: "d9"} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, c.report)) - if len(store.heard) != 1 { - t.Errorf("%s was set aside as stale, and the mesh will never hear it again: %+v", - c.what, *to) - } - } -} diff --git a/internal/link/watched.go b/internal/link/watched.go index 38bb8e5c..2a7c3003 100644 --- a/internal/link/watched.go +++ b/internal/link/watched.go @@ -172,9 +172,6 @@ func (p PowerState) Away() bool { return p.State == "sleeping" || p.State == "sh // one it holds (mesh-host `refuseOlder`, novox/hq issue 107): the receiver's refusal S13 counts. const StaleRefusal = "is older than what the mesh last said to this node" -// IsStaleRefusal says a report's refusal is the node-engine refusing a declaration older than it holds. -func IsStaleRefusal(refused string) bool { return strings.Contains(refused, StaleRefusal) } - // A Refusal is one receiver refusing something older than what it holds (novox/hq to-be 45 §6, ADR 0227 // rule 2): a machine refusing a declaration, or this controller refusing an account. type Refusal struct { @@ -189,15 +186,6 @@ type Refusal struct { At time.Time } -// WriterEpoch is how a refused declaration's writer is said before it is named: by the epoch it -// claimed, or as a controller that claimed none. -func WriterEpoch(epoch int64) string { - if epoch > 0 { - return fmt.Sprintf("the controller of epoch %d", epoch) - } - return "a controller that claimed no epoch" -} - // WriterNodeEngine is a machine's node-engine as the writer of an account the controller refused. func WriterNodeEngine(node string) string { return "the node-engine on " + node } diff --git a/module.json b/module.json index d41ce99e..eb6ee64c 100644 --- a/module.json +++ b/module.json @@ -25,23 +25,6 @@ }, "secrets-owner": "mesh-controller", "prepares": true, - "settings": { - "merge-window": { - "kind": "preference", - "default": "90s", - "why": "merges are collected while they keep coming, and a batch is planned once no merge came for this long (novox/hq ADR 0276)" - }, - "merge-window-at-most": { - "kind": "preference", - "default": "10m", - "why": "a busy period never holds a batch longer than this after its first merge (novox/hq ADR 0276)" - }, - "push-wait": { - "kind": "preference", - "default": "2m", - "why": "how long a push to a named machine waits for that machine's answer, so its call says how it went; a call's own wait overrides it, and 0 waits for nothing (novox/hq issue 629)" - } - }, "tools": [ "tools", "calls", @@ -124,23 +107,6 @@ "place": "mesh", "owner": "mesh-controller" }, - { - "id": "merge-window", - "type": "file", - "path": "${dir:mesh-state}/merge-window.json", - "mode": "0600", - "owner": "mesh-controller", - "content": "{\n \"merge-window\": \"${setting:merge-window}\",\n \"merge-window-at-most\": \"${setting:merge-window-at-most}\"\n}\n" - }, - { - "id": "push", - "trusted": false, - "type": "file", - "path": "${dir:mesh-state}/push.json", - "mode": "0600", - "owner": "mesh-controller", - "content": "{\n \"push-wait\": \"${setting:push-wait}\"\n}\n" - }, { "id": "controller", "type": "process", @@ -164,9 +130,7 @@ "MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}", "MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}", "MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus", - "MESH_CONTROLLER_VERSION": "${version}", - "MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json", - "MESH_PUSH_FILE": "${dir:mesh-state}/push.json" + "MESH_CONTROLLER_VERSION": "${version}" }, "replaces": [ "server" diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index c4cff530..7e778f5a 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -7,9 +7,9 @@ The copy carries the catalogue's private details: domains, first names in module private addresses. That is acceptable while mesh-controller is private; if it ever goes public, this copy is a second place to clean besides the catalogue itself. -mesh-catalog 2246ccc492e59579a140752e6f574b2121d296af modules/*/module.json, modules/*/files, modules/*/templates, predefined.json +mesh-catalog 3a1017f5594217092a654dd1e2d72226766bc16a modules/*/module.json, modules/*/files, modules/*/templates, predefined.json mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk 34f210e5eba1d16abe06b526de1026c282692b83 conformance/events/module-event.json +mesh-sdk 748c564cdf7cd2cad57073cdf54bd752c1947025 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/testdata/beside/mesh-catalog/modules/betterbird/files/accounts.json b/testdata/beside/mesh-catalog/modules/betterbird/files/accounts.json deleted file mode 100644 index e182cc3d..00000000 --- a/testdata/beside/mesh-catalog/modules/betterbird/files/accounts.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "accounts": {} -} diff --git a/testdata/beside/mesh-catalog/modules/betterbird/files/betterbird b/testdata/beside/mesh-catalog/modules/betterbird/files/betterbird deleted file mode 100755 index 3ce088a8..00000000 --- a/testdata/beside/mesh-catalog/modules/betterbird/files/betterbird +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# Betterbird's start, written by the mesh (module betterbird, novox/hq ADR 0316). Replaced at every push. -# -# Before Betterbird starts, the module's tools place its user.js in the profile (the settings prefs, accounts, identities, calendars, -# address-books and tags) and write what Betterbird rewrites itself: the account lists and each declared account's filters. They do -# it only while no Betterbird of this account runs, because Betterbird rewrites its saved preferences when it quits; a -# second start while one runs only hands over to it. What the preparation could not do it says here, and -# betterbird_check says it too. Betterbird starts either way: its configuration never holds the mail back. -/usr/local/lib/mesh-betterbird/betterbird-tools prepare || echo "betterbird: the mesh's preparation did not complete; betterbird_check says why" >&2 -exec /opt/betterbird/betterbird "$@" diff --git a/testdata/beside/mesh-catalog/modules/betterbird/files/configuration.json b/testdata/beside/mesh-catalog/modules/betterbird/files/configuration.json deleted file mode 100644 index 70204462..00000000 --- a/testdata/beside/mesh-catalog/modules/betterbird/files/configuration.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "identities": {}, - "calendars": {}, - "address-books": {}, - "filters": {} -} diff --git a/testdata/beside/mesh-catalog/modules/betterbird/files/eu.betterbird.Betterbird.desktop b/testdata/beside/mesh-catalog/modules/betterbird/files/eu.betterbird.Betterbird.desktop deleted file mode 100644 index e409fcf8..00000000 --- a/testdata/beside/mesh-catalog/modules/betterbird/files/eu.betterbird.Betterbird.desktop +++ /dev/null @@ -1,25 +0,0 @@ -[Desktop Entry] -Version=1.0 -Type=Application -Name=Betterbird -GenericName=Mail Client -Comment=Send and receive mail with Betterbird -Exec=/usr/local/bin/betterbird %u -Icon=/opt/betterbird/chrome/icons/default/default128.png -Terminal=false -Categories=Network;Email; -MimeType=message/rfc822;x-scheme-handler/mailto;application/x-xpinstall;application/x-extension-ics;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid; -StartupNotify=true -StartupWMClass=eu.betterbird.Betterbird -Keywords=Email;E-mail;Calendar;Thunderbird; -Actions=ComposeMessage;OpenAddressBook; -# Written by the mesh (module betterbird, novox/hq ADR 0316 §3), in place of the community package's entry: -# it starts Betterbird through the module's start, and names the icon in the installation, with no link. - -[Desktop Action ComposeMessage] -Name=Write new message -Exec=/usr/local/bin/betterbird -compose - -[Desktop Action OpenAddressBook] -Name=Open address book -Exec=/usr/local/bin/betterbird -addressbook diff --git a/testdata/beside/mesh-catalog/modules/betterbird/files/prefs.json b/testdata/beside/mesh-catalog/modules/betterbird/files/prefs.json deleted file mode 100644 index 45fd9732..00000000 --- a/testdata/beside/mesh-catalog/modules/betterbird/files/prefs.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "prefs": {} -} diff --git a/testdata/beside/mesh-catalog/modules/betterbird/files/tags.json b/testdata/beside/mesh-catalog/modules/betterbird/files/tags.json deleted file mode 100644 index 4772f099..00000000 --- a/testdata/beside/mesh-catalog/modules/betterbird/files/tags.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "tags": {} -} diff --git a/testdata/beside/mesh-catalog/modules/betterbird/module.json.captured b/testdata/beside/mesh-catalog/modules/betterbird/module.json.captured deleted file mode 100644 index 891f8832..00000000 --- a/testdata/beside/mesh-catalog/modules/betterbird/module.json.captured +++ /dev/null @@ -1,253 +0,0 @@ -{ - "module": "betterbird", - "version": "1", - "upgrade": { - "policy": "record", - "why": "a new build may remove the community package betterbird-bin and replaces the installation in /opt/betterbird, which a running Betterbird has open: the switch on each node and every upgrade wait for the operator, who quits Betterbird first and then pushes (novox/hq ADR 0316 §4, §5)" - }, - "requires": [ - "x11-display" - ], - "invokes": [ - "seat:mesh-controller.settings", - "seat:mesh-controller.nodes", - "seat:mesh-controller.node", - "betterbird.betterbird_derive" - ], - "tools": [ - "betterbird_status", - "betterbird_prefs", - "betterbird_config", - "betterbird_config_apply", - "betterbird_user_js", - "betterbird_log", - "betterbird_restart", - "betterbird_check", - "betterbird_accounts", - "betterbird_identities", - "betterbird_calendars", - "betterbird_address_books", - "betterbird_filters", - "betterbird_tags", - "betterbird_derive", - "betterbird_drift", - "betterbird_reminder_test" - ], - "config": { - "prefs": { - "type": "object", - "description": "Betterbird's preferences the operator sets, by their own names (mailnews.*, calendar.* and the like), laid into prefs.json; an account's, a server's and a tag's are refused, as accounts, the four kinds and tags own them (novox/hq ADR 0329 §6).", - "trusted": true, - "default": {} - }, - "accounts": { - "type": "object", - "description": "The mail accounts Betterbird is set up with, each by a name of the operator's: its address, its sender's name and its incoming and outgoing servers; no password, token or cookie (novox/hq ADR 0329 §2).", - "trusted": true, - "default": {} - }, - "identities": { - "type": "object", - "description": "The identities beside an account's own, each by a name of the operator's: the address and name it sends as, its signature as a list of lines, and the profile's key to adopt on a node.", - "trusted": true, - "default": {} - }, - "calendars": { - "type": "object", - "description": "The calendars Betterbird shows, each by a name of the operator's: where it is and how it is reached, and the profile's key to adopt on a node.", - "trusted": true, - "default": {} - }, - "address-books": { - "type": "object", - "description": "The address books Betterbird uses, each by a name of the operator's: where it is and how it is reached, and the profile's key to adopt on a node.", - "trusted": true, - "default": {} - }, - "filters": { - "type": "object", - "description": "The message filters of each account, by account: what each matches and what it does with the mail.", - "trusted": true, - "default": {} - }, - "tags": { - "type": "object", - "description": "The tags Betterbird offers, each by its own key: its name, colour and order; trusted since issue 751, so a change to it is the operator's to approve.", - "trusted": true, - "default": {} - } - }, - "resources": [ - { - "id": "configuration-dir", - "type": "directory", - "path": "${machine:account-home}/.config/betterbird", - "owner": "${machine:account}", - "mode": "0755", - "unknown": "move-out", - "knows": ["accounts-record.json", "configuration-record.json", "kept/"] - }, - { - "id": "prefs", - "type": "file", - "path": "${machine:account-home}/.config/betterbird/prefs.json", - "owner": "${machine:account}", - "mode": "0644", - "merge": "json", - "takes-config": [ - "prefs" - ], - "template": "files/prefs.json", - "trusted": true - }, - { - "id": "accounts", - "type": "file", - "path": "${machine:account-home}/.config/betterbird/accounts.json", - "owner": "${machine:account}", - "mode": "0600", - "merge": "json", - "takes-config": [ - "accounts" - ], - "template": "files/accounts.json", - "trusted": true - }, - { - "id": "configuration", - "type": "file", - "path": "${machine:account-home}/.config/betterbird/configuration.json", - "owner": "${machine:account}", - "mode": "0600", - "merge": "json", - "takes-config": [ - "identities", - "calendars", - "address-books", - "filters" - ], - "template": "files/configuration.json", - "trusted": true - }, - { - "id": "tags", - "type": "file", - "path": "${machine:account-home}/.config/betterbird/tags.json", - "owner": "${machine:account}", - "mode": "0644", - "merge": "json", - "takes-config": [ - "tags" - ], - "template": "files/tags.json", - "trusted": true - }, - { - "id": "community-package", - "type": "package", - "package": "betterbird-bin", - "absent": true - }, - { - "id": "installation", - "type": "archive", - "path": "/opt/betterbird", - "artifact": "archive", - "unknown": "program" - }, - { - "id": "owl", - "type": "archive", - "path": "/opt/betterbird/owl", - "artifact": "owl", - "unknown": "program" - }, - { - "id": "policies", - "type": "archive", - "path": "/opt/betterbird/distribution", - "artifact": "distribution", - "unknown": "program" - }, - { - "id": "spelling", - "type": "archive", - "path": "/opt/betterbird/defaults/pref", - "artifact": "defaults", - "unknown": "program" - }, - { - "id": "preparation", - "type": "archive", - "path": "/usr/local/lib/mesh-betterbird", - "artifact": "tools", - "unknown": "program" - }, - { - "id": "start", - "type": "file", - "path": "/usr/local/bin/betterbird", - "mode": "0755", - "template": "files/betterbird" - }, - { - "id": "launcher", - "type": "file", - "path": "/usr/local/share/applications/eu.betterbird.Betterbird.desktop", - "mode": "0644", - "template": "files/eu.betterbird.Betterbird.desktop" - } - ], - "build": { - "artifacts": [ - { - "name": "tools", - "kind": "bundle", - "language": "go", - "system": "arch", - "from": "cmd/betterbird-tools", - "binary": "betterbird-tools", - "loads": [ - "betterbird-tools" - ] - }, - { - "name": "archive", - "kind": "archive", - "url": "https://www.betterbird.eu/downloads/LinuxArchive/betterbird-153.4.0esr-bb10.en-US.linux-x86_64.tar.xz", - "sha256": "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2", - "version": "153.4.0esr-bb10", - "within": "betterbird", - "names-on-purpose": { - "www.betterbird.eu": "Betterbird's project publishes its Linux release there; the build seat fetches it once, by its pinned sha256 (novox/hq ADR 0316 §2)" - } - }, - { - "name": "owl", - "kind": "archive", - "url": "https://www.beonex.com/owl/owl.xpi", - "sha256": "a54cc3d52113b34f97942ee19190947dee23561e2217f18db3d80ac023991221", - "version": "1.5.6.1", - "file": "owl.xpi", - "names-on-purpose": { - "www.beonex.com": "Owl's vendor publishes its xpi there; the build seat fetches it once, by its pinned sha256 (novox/hq ADR 0316 §6)" - } - }, - { - "name": "distribution", - "kind": "archive", - "from": "install/distribution" - }, - { - "name": "defaults", - "kind": "archive", - "from": "install/defaults-pref" - }, - { - "name": "files", - "kind": "archive", - "from": "files" - } - ] - } -} diff --git a/testdata/beside/mesh-catalog/modules/gitea/files/config.json b/testdata/beside/mesh-catalog/modules/gitea/files/config.json index 0967ef42..7ee627f8 100644 --- a/testdata/beside/mesh-catalog/modules/gitea/files/config.json +++ b/testdata/beside/mesh-catalog/modules/gitea/files/config.json @@ -1 +1 @@ -{} +{"main-mergers": ["jschoubben"]} diff --git a/testdata/beside/mesh-catalog/modules/gitea/files/repositories.json b/testdata/beside/mesh-catalog/modules/gitea/files/repositories.json index 36626ccc..a8293008 100644 --- a/testdata/beside/mesh-catalog/modules/gitea/files/repositories.json +++ b/testdata/beside/mesh-catalog/modules/gitea/files/repositories.json @@ -1 +1 @@ -{"repositories": ${config:repositories}} +{"repositories": ${config:repositories}, "keep-main-rules": ${config:keep-main-rules}} diff --git a/testdata/beside/mesh-catalog/modules/gitea/module.json.captured b/testdata/beside/mesh-catalog/modules/gitea/module.json.captured index 31bbb265..1ac1aeb8 100644 --- a/testdata/beside/mesh-catalog/modules/gitea/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/gitea/module.json.captured @@ -169,11 +169,23 @@ "trusted": true, "default": {} }, + "keep-main-rules": { + "type": "boolean", + "description": "Whether the module keeps main's branch protection as the mesh wants it. True: every ten minutes, and at the start, main's rule in every repository of the organisation novox is put to no push and the tools' account alone merging, and each apply of the `repositories` value narrows the main rule of the repositories it lists, requiring no status. False (the default): the module changes no existing repository's branch protection, whoever set it at the forge, but takes away a required status it finds on a rule covering main, once (ADR 0534 §1); a repository listed in `repositories` that does not exist is still made, with its first protection.", + "trusted": true, + "default": false + }, "repositories": { "type": "object", - "description": "The repositories of the organisation novox the mesh holds (ADR 0428): each by name, {\"\": {\"private\": true, \"description\": \"...\"}}. One that does not exist is made by the module's setup login under its own account, its main protected and given a first commit (a README naming the description), its rule narrowed to no push and the tools' merge requiring mesh/check, and only then moved into the organisation; one that exists only has main's rule put right, mesh/check in place of mesh/merge-gate and mesh/repo-check and its other statuses kept (a main requiring mesh/repo-check alone keeps it). Every other repository whose main requires mesh/merge-gate has the two swapped for mesh/check; nothing else of a repository out of the value is touched.", + "description": "The repositories of the organisation novox the mesh holds (ADR 0428): each by name, {\"\": {\"private\": true, \"description\": \"...\"}}. One that does not exist is made by the module's setup login under its own account, its main protected and given a first commit (a README naming the description ), its rule narrowed to no push, the tools' account and main-mergers alone merging, and no status required (ADR 0534), and only then moved into the organisation; one that exists only has main's rule put right so. Every other repository has the statuses its main requires taken off; nothing else of a repository out of the value is touched. With keep-main-rules false a listed repository that exists likewise has only its required statuses taken off, once.", "trusted": true, "default": {} + }, + "main-mergers": { + "type": "list", + "description": "The forge logins that may merge into main beside the tools' account, on every repository of the organisation (ADR 0534 §2): the operator's own, so the operator can always merge by hand at the forge. The tools' account is always on main's merge whitelist; no agent's account belongs here.", + "trusted": true, + "default": ["jschoubben"] } }, "facts": { @@ -302,7 +314,9 @@ "template": "files/config.json", "merge": "json", "takes-config": [ - "url" + "url", + "keep-main-rules", + "main-mergers" ] }, { @@ -365,10 +379,12 @@ "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", "MESH_GITEA_TOOLS_USER": "mesh-tools", "MESH_REPOSITORIES_FILE": "${dir:mesh-state}/repositories.json", - "MESH_REPOSITORIES_OUTCOME": "${dir:runtime-state}/repositories-outcome.json" + "MESH_REPOSITORIES_OUTCOME": "${dir:runtime-state}/repositories-outcome.json", + "MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json" }, "restart-on": [ - "repositories-file" + "repositories-file", + "runtime-config" ] } ], diff --git a/testdata/beside/mesh-catalog/modules/mesh-build-go/module.json.captured b/testdata/beside/mesh-catalog/modules/mesh-build-go/module.json.captured index db0dbe1c..4d2b080a 100644 --- a/testdata/beside/mesh-catalog/modules/mesh-build-go/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/mesh-build-go/module.json.captured @@ -51,7 +51,8 @@ "requires": [ "forge-account", "go-toolchain", - "artifact-store" + "artifact-store", + "node-tools" ], "own-secrets": { "broker": "${dir:state}/broker" diff --git a/testdata/beside/mesh-catalog/modules/mesh-build-image/module.json.captured b/testdata/beside/mesh-catalog/modules/mesh-build-image/module.json.captured index 2678972c..cfe8489f 100644 --- a/testdata/beside/mesh-catalog/modules/mesh-build-image/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/mesh-build-image/module.json.captured @@ -39,7 +39,8 @@ "requires": [ "forge-account", "rootless-image-build", - "artifact-store" + "artifact-store", + "node-tools" ], "own-secrets": { "broker": "${dir:state}/broker" diff --git a/testdata/beside/mesh-catalog/modules/mesh-build-node/module.json.captured b/testdata/beside/mesh-catalog/modules/mesh-build-node/module.json.captured index c3b1cf84..25f858eb 100644 --- a/testdata/beside/mesh-catalog/modules/mesh-build-node/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/mesh-build-node/module.json.captured @@ -50,7 +50,8 @@ "forge-account", "npm-package-registry", "node-toolchain", - "artifact-store" + "artifact-store", + "node-tools" ], "contributes": { "npm-package-registry": { diff --git a/testdata/beside/mesh-catalog/modules/mesh-delivery-gate/module.json.captured b/testdata/beside/mesh-catalog/modules/mesh-delivery-gate/module.json.captured index 086ad999..74da6dc0 100644 --- a/testdata/beside/mesh-catalog/modules/mesh-delivery-gate/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/mesh-delivery-gate/module.json.captured @@ -8,18 +8,13 @@ "scope": "mesh", "serves": [ "deliveries", - "times", "show", "groups", - "what-if", "checks", "table", "stalled", "recheck", - "release", - "stop", - "close", - "retire-history" + "close" ] } ], @@ -30,7 +25,6 @@ "gitea.forge.status", "gitea.forge.pull-request.edited", "gitea.forge.pull-request.closed", - "mesh-controller.plan-moved", "mesh-delivery-forge-bridge.forge.**", "mesh-config.config-changed" ], @@ -40,11 +34,6 @@ "merged" ], "invokes": [ - "seat:mesh-controller.deliver", - "seat:mesh-controller.delivery-stop", - "seat:mesh-controller.delivery-backups-again", - "seat:mesh-controller.delivery-walks", - "seat:mesh-controller.plans", "gitea.gitea_commit_statuses", "gitea.gitea_contains", "gitea.gitea_open_pulls" @@ -71,13 +60,8 @@ } ], "reads": [ - "mesh-delivery.deliveries", - "mesh-delivery.groups", - "mesh-delivery.pulls", "mesh-delivery-teller.answers", - "mesh-delivery-checker.verdicts", - "mesh-controller.assignments", - "node-engine.have" + "mesh-delivery-checker.verdicts" ], "tools": [ "delivery_status" diff --git a/testdata/beside/mesh-catalog/modules/mesh-issues/files/settings.json b/testdata/beside/mesh-catalog/modules/mesh-issues/files/settings.json index 1cb2d002..0231a55d 100644 --- a/testdata/beside/mesh-catalog/modules/mesh-issues/files/settings.json +++ b/testdata/beside/mesh-catalog/modules/mesh-issues/files/settings.json @@ -9,7 +9,6 @@ "mesh-catalog modules/audit-logger", "mesh-catalog modules/avahi", "mesh-catalog modules/baserow", - "mesh-catalog modules/betterbird", "mesh-catalog modules/blueman", "mesh-catalog modules/bluetooth", "mesh-catalog modules/brother", diff --git a/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured b/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured index 830e1495..ee14150a 100644 --- a/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/mesh-issues/module.json.captured @@ -57,13 +57,15 @@ "consumes": [ "mesh-issues.opened", "mesh-issues.moved", - "mesh-delivery-gate.transition" + "mesh-delivery-gate.transition", + "mesh-delivery-view.delivered" ], "invokes": [ "claude-code.claude_code_judge", "seat:mesh-delivery.checks", "seat:operator-channel.notify", "gitea.gitea_pull_reviews", + "gitea.gitea_contains", "records.records_status", "records.records_list", "records.records_read", @@ -102,7 +104,6 @@ "mesh-catalog modules/audit-logger", "mesh-catalog modules/avahi", "mesh-catalog modules/baserow", - "mesh-catalog modules/betterbird", "mesh-catalog modules/blueman", "mesh-catalog modules/bluetooth", "mesh-catalog modules/brother", diff --git a/testdata/beside/mesh-catalog/modules/messenger/module.json.captured b/testdata/beside/mesh-catalog/modules/messenger/module.json.captured index d7da1d68..4aceaebe 100644 --- a/testdata/beside/mesh-catalog/modules/messenger/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/messenger/module.json.captured @@ -77,7 +77,9 @@ "mesh-controller.condition-raised", "mesh-controller.condition-changed", "mesh-controller.condition-cleared", - "mesh-controller.plan-moved", + "mesh-delivery-view.moving", + "mesh-delivery-view.settled", + "mesh-delivery-view.stuck", "intake.choice", "intake.link", "intake.failed", diff --git a/testdata/beside/mesh-catalog/modules/restic/module.json.captured b/testdata/beside/mesh-catalog/modules/restic/module.json.captured index 18ee811d..ea5ec8b9 100644 --- a/testdata/beside/mesh-catalog/modules/restic/module.json.captured +++ b/testdata/beside/mesh-catalog/modules/restic/module.json.captured @@ -13,6 +13,9 @@ ] } ], + "emits": [ + "backed-up" + ], "tools": [ "backup_health", "backup_delete_retired",