Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ecbb317dc3 | ||
|
|
dd2d8b91be | ||
|
|
8831c74f70 | ||
|
|
a96bd2ff38 |
@@ -112,13 +112,11 @@ func acceptForProvision(ctx context.Context, provider, providerModule, provision
|
||||
if len(accepted.Sealed) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Composed into each consumer's composition (novox/hq ADR 0539 clause 5): a new revision its want then names.
|
||||
if _, err := composeModules(ctx, open, accepted.Machines); err != nil {
|
||||
return fmt.Errorf("%s is sealed and its compositions on %s could not be written: %w", provision,
|
||||
strings.Join(accepted.Machines, ", "), err)
|
||||
if err := sendTo(ctx, open, accepted.Machines); err != nil {
|
||||
return fmt.Errorf("%s is sealed and could not be sent: %w; `push %s` sends it", provision, err,
|
||||
strings.Join(accepted.Machines, "`, `push "))
|
||||
}
|
||||
fmt.Printf(" composed for %s; each consumer starts again on it once its want names the new composition\n",
|
||||
strings.Join(accepted.Machines, ", "))
|
||||
fmt.Printf(" sent %s; each consumer starts again on it\n", strings.Join(accepted.Machines, ", "))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+17
-18
@@ -58,6 +58,13 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
|
||||
// what the reads before it would say about a machine left out is not the answer's.
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
|
||||
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
|
||||
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
|
||||
@@ -164,21 +171,8 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
|
||||
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
}
|
||||
}
|
||||
return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// composedAfterAct composes the node an assignment or its end changed (novox/hq ADR 0539 clause 5): what the
|
||||
// answer says of it. The assigner then writes the node's wants by its rules; nothing is sent.
|
||||
func composedAfterAct(ctx context.Context, open *stores, node string) string {
|
||||
wrote, err := composeModules(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return fmt.Sprintf("\n but its compositions on %s could not all be written: %v", node, firstLine(err.Error()))
|
||||
}
|
||||
if len(wrote[node]) == 0 {
|
||||
return fmt.Sprintf("\n %s's compositions were already as composed now", node)
|
||||
}
|
||||
return fmt.Sprintf("\n composed for %s (%d composition(s) written): its wants follow by the assigner's rules", node,
|
||||
len(wrote[node]))
|
||||
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||
@@ -237,6 +231,11 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
}
|
||||
// One resolution after the write, as assign's answer (novox/hq issue 237).
|
||||
ctx = leftOutSaidBy(ctx, io.Discard)
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -274,8 +273,8 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s: its want is written removed by the assigner",
|
||||
node, strings.Join(modules, ", "))
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
after := resolvedAfter(ctx, open, shelf, node)
|
||||
for _, line := range unheldChangeTo(shelf, node, assigned, after.modules, after.unheld) {
|
||||
answer += "\n " + line
|
||||
@@ -296,7 +295,7 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
return answer + composedAfterAct(ctx, open, node) + blockedElsewhere(ctx, open, node), nil
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// afterAct is the one resolution of a node an act's answer is said from, taken after its write
|
||||
|
||||
@@ -6,10 +6,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -54,7 +50,6 @@ func anAdoptedAnchor(t *testing.T) (*stores, *[]string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
onTheBusWithItsHaves(t, open)
|
||||
sent := &[]string{}
|
||||
saved := sendNodes
|
||||
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
||||
@@ -88,6 +83,13 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
||||
Firewall: "ufw", Held: held, Reachable: reachable,
|
||||
@@ -176,7 +178,7 @@ func TestTakingNamesWhatItReplaces(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
||||
!strings.Contains(said, "compositions of anchor are written") {
|
||||
!strings.Contains(said, "push anchor") {
|
||||
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -248,7 +250,6 @@ func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T)
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*sent = nil // the take composed the node; what follows is judged from here
|
||||
reportsHolding(t, open, heldFile)
|
||||
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
@@ -437,7 +438,6 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*sent = nil // the take composed the node; what follows is judged from here
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
@@ -495,6 +495,52 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
||||
// composed after it — never sent after it with the node still adopted. And the send inside the
|
||||
// flip is not made to wait on the flip's own hold.
|
||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var heldElsewhere, heldHere error
|
||||
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
||||
// Another caller cannot hold the node while the flip sends it.
|
||||
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||
defer cancel()
|
||||
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
||||
release()
|
||||
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
||||
}
|
||||
// The flip's own send holds it without waiting on itself.
|
||||
_, release, err := holdNodes(inner, open, names)
|
||||
if err != nil {
|
||||
heldHere = err
|
||||
return err
|
||||
}
|
||||
release()
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if heldElsewhere != nil || heldHere != nil {
|
||||
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
||||
}
|
||||
// And given back once it is done.
|
||||
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||
// not only its files, and the digest changes when any of them does.
|
||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||
@@ -546,7 +592,6 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*sent = nil // the take composed the node; what follows is judged from here
|
||||
// Only a loopback listener: nothing off the machine, which is the same silence.
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
||||
@@ -567,94 +612,32 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// onTheBusWithItsHaves gives the controller a bus that holds, for the anchor, a want and a have of its module that agree: the node
|
||||
// has everything wanted of it, as read from the haves (novox/hq ADR 0539 §9), so a flip is refused for what it is about and
|
||||
// not because the bus could not be read.
|
||||
func onTheBusWithItsHaves(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
||||
// holds the node, so it waits for whatever is converging it.
|
||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
composed := "sha256:" + strings.Repeat("b", 64)
|
||||
put := func(bucket, key string, value []byte) {
|
||||
kv, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
wantKey, err := sdkdelivery.WantKey("anchor", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
haveKey, err := sdkdelivery.HaveKey("anchor", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, err := sdkdelivery.EncodeWant(sdkdelivery.Want{FP: fp, Composed: composed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have, err := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: fp, Composed: composed, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(sdkdelivery.AssignerWantsBucket, wantKey, want)
|
||||
put(sdkdelivery.HaveBucket, haveKey, have)
|
||||
}
|
||||
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
||||
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
||||
|
||||
// **When the bus cannot be read, whether the machine runs what is wanted is not known, and the flip says exactly that** (issue
|
||||
// 1945): not "push", and not "has not reported".
|
||||
func TestTheFlipIsRefusedWithTheBusUnreachableSayingItCannotTell(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
servingBus.Store(nil) // this process has no bus now, and no address to dial
|
||||
_, err := converge(t.Context(), open, "anchor", true, "", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "cannot tell whether anchor runs what is wanted: the bus is not reachable") ||
|
||||
strings.Contains(err.Error(), "push") {
|
||||
t.Fatalf("the flip with the bus out of reach: %v", err)
|
||||
var whileFlipping error
|
||||
sendNodes = func(context.Context, *stores, []string) error {
|
||||
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
||||
return nil
|
||||
}
|
||||
if len(*sent) != 0 {
|
||||
t.Fatal("a refused flip sent something")
|
||||
}
|
||||
}
|
||||
|
||||
// **A take composes the node only when it is performed** (issue 1945): a preview, a take that names no digest and a take refused
|
||||
// for a changed preview compose nothing; the performed take composes the machine once, and says so.
|
||||
func TestATakeComposesTheNodeOnlyWhenItIsPerformed(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||
if err != nil {
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil {
|
||||
t.Fatal("a take of a held module without the preview's digest was performed")
|
||||
}
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: "not-the-digest"}); err == nil {
|
||||
t.Fatal("a take naming another preview's digest was performed")
|
||||
}
|
||||
if len(*sent) != 0 {
|
||||
t.Fatalf("a preview or a refused take composed %v", *sent)
|
||||
}
|
||||
said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: takeDigestIn(t, preview)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*sent) != 1 || (*sent)[0] != "anchor" || !strings.Contains(said, "compositions of anchor are written") {
|
||||
t.Fatalf("the performed take composed %v and said:\n%s", *sent, said)
|
||||
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
||||
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,9 +205,9 @@ func adoptedNodes(nodes []inventory.Node) []string {
|
||||
// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the
|
||||
// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035).
|
||||
|
||||
// sendNodes composes the named machines as they should be now (novox/hq ADR 0539 clause 5): each node's want then
|
||||
// names what changed. A variable so a test can see what an act would compose without a broker.
|
||||
var sendNodes = composeAndSay
|
||||
// sendNodes sends the named machines what they should be now. A variable so a test can see what
|
||||
// an act would send without a broker.
|
||||
var sendNodes = sendTo
|
||||
|
||||
// DefaultFilter is the module converging a node assigns to load the mesh's derived filter.
|
||||
const DefaultFilter = "nftables"
|
||||
@@ -276,8 +276,8 @@ func take(ctx context.Context, open *stores, node, module string, opts takeOptio
|
||||
// is still the machine: the same two refusals the flip makes.
|
||||
if age := time.Since(c.reported.At); age > reportFreshFor {
|
||||
return preview, fmt.Errorf("%s last said what it holds %s ago, and a take acts only on "+
|
||||
"an account newer than %s: wait for its next report, then preview again",
|
||||
node, age.Round(time.Second), reportFreshFor)
|
||||
"an account newer than %s: run `push %s --wait 2m`, then preview again",
|
||||
node, age.Round(time.Second), reportFreshFor, node)
|
||||
}
|
||||
if opts.Digest == "" {
|
||||
return preview, fmt.Errorf("taking %s on %s acts on the preview you saw: name its digest, "+
|
||||
@@ -294,12 +294,9 @@ func take(ctx context.Context, open *stores, node, module string, opts takeOptio
|
||||
}
|
||||
said := fmt.Sprintf("%s is taken on %s", module, node)
|
||||
if holds {
|
||||
said += "; its next composition replaces what the node found and holds for it:\n" + preview
|
||||
said += "; the next push replaces what the node found and holds for it:\n" + preview
|
||||
}
|
||||
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||
return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err
|
||||
}
|
||||
return said + fmt.Sprintf("\n the compositions of %s are written; its wants then cut it over", node), nil
|
||||
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
|
||||
}
|
||||
|
||||
// comparison is everything a take puts beside what the module declares: the machine's account of
|
||||
@@ -436,7 +433,7 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
|
||||
fmt.Fprintf(&b, " settings from %s: %s\n", layer.From, strings.Join(keys, ", "))
|
||||
}
|
||||
if c.settingsRefused != "" {
|
||||
fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the composition leaves it out: %s\n", c.settingsRefused)
|
||||
fmt.Fprintf(&b, " SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: %s\n", c.settingsRefused)
|
||||
}
|
||||
preview = strings.TrimRight(b.String(), "\n")
|
||||
sum := sha256.Sum256([]byte(preview))
|
||||
@@ -611,6 +608,16 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
if filter == "" {
|
||||
filter = DefaultFilter
|
||||
}
|
||||
if yes {
|
||||
// Held from the checks to the send, so no push composed before the flip is sent after it
|
||||
// and returns the node to adopted.
|
||||
held, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
ctx = held
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -618,27 +625,24 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
if !record.Adopted {
|
||||
return "", fmt.Errorf("%s is converged already; there is nothing to flip", node)
|
||||
}
|
||||
reports, err := lastReports(ctx, inv)
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
current, unknown := false, false
|
||||
current := false
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
current, unknown = r.Current, r.CurrentUnknown
|
||||
current = r.Current
|
||||
}
|
||||
}
|
||||
if unknown {
|
||||
return "", fmt.Errorf("cannot tell whether %s runs what is wanted: the bus is not reachable", node)
|
||||
}
|
||||
reported, err := inv.AdoptionOf(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !current || reported.At.IsZero() {
|
||||
return "", fmt.Errorf("%s does not run everything that is wanted of it yet, or has not reported, so what it "+
|
||||
"says is reachable may not be the machine as it is: converge once its haves match its wants and it has "+
|
||||
"reported", node)
|
||||
return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+
|
||||
"says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+
|
||||
"converge once it has applied", node, node)
|
||||
}
|
||||
|
||||
assignedWhenPreviewed, err := inv.Assigned(ctx, node)
|
||||
@@ -734,13 +738,13 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
if yes && countReachable(reported) == 0 {
|
||||
return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+
|
||||
"up ever is: its account looks partial — whatever reads what is listening, or what "+
|
||||
"the container runtime publishes, did not answer. Fix that on the machine, wait for its next report, "+
|
||||
"then preview again", node)
|
||||
"the container runtime publishes, did not answer. Fix that on the machine and run "+
|
||||
"`push %s --wait 2m`, then preview again", node, node)
|
||||
}
|
||||
if age := time.Since(reported.At); age > reportFreshFor {
|
||||
return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+
|
||||
"only on an account newer than %s: wait for its next report, then preview again", node,
|
||||
age.Round(time.Second), reportFreshFor)
|
||||
"only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+
|
||||
"then preview again", node, age.Round(time.Second), reportFreshFor, node)
|
||||
}
|
||||
if digest == "" {
|
||||
return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+
|
||||
@@ -783,7 +787,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
|
||||
said += "; took " + strings.Join(took, ", ")
|
||||
}
|
||||
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||
return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err
|
||||
return said + "\n and it could not be sent: run `push " + node + "`", err
|
||||
}
|
||||
return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil
|
||||
}
|
||||
@@ -1027,12 +1031,18 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) {
|
||||
if record.Adopted {
|
||||
return "", fmt.Errorf("%s is adopted already", node)
|
||||
}
|
||||
held, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
ctx = held
|
||||
if err := inv.SetAdopted(ctx, node, true); err != nil {
|
||||
return "", err
|
||||
}
|
||||
said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node)
|
||||
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||
return said + "\n and its compositions could not be written yet: the next act that composes them carries it", err
|
||||
return said + "\n and it could not be sent: run `push " + node + "`", err
|
||||
}
|
||||
return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// busWaitBound is how long a module whose ask the bus did not answer is asked again at every look before the plan fails
|
||||
// (issue 1334).
|
||||
const busWaitBound = 10 * time.Minute
|
||||
|
||||
// busWaiting says an open plan has a module of its current tier that the bus did not answer an ask of: the look must come back
|
||||
// for it, as nothing else will when no build of the tier is in flight (issue 1334).
|
||||
func busWaiting(p inventory.Plan) bool {
|
||||
return len(busWaitingIn(p)) > 0
|
||||
}
|
||||
|
||||
// busWaitingIn are the modules of the plan's current tier left unasked because the bus did not answer, sorted.
|
||||
func busWaitingIn(p inventory.Plan) []string {
|
||||
if !p.Open() || p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "" && s.BusWaitFrom != nil {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// busWaitWords is how a verb says modules are waiting for the bus: not asked yet, and asked again by the plan at its next look.
|
||||
func busWaitWords(p inventory.Plan, modules []string) string {
|
||||
return fmt.Sprintf("%s not asked yet: the bus did not answer; %s asks again at its next look", strings.Join(modules, ", "), p.ID)
|
||||
}
|
||||
|
||||
// slicesWithout is names without those in drop, in order.
|
||||
func slicesWithout(names, drop []string) []string {
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
if !slices.Contains(drop, n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting:
|
||||
// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue
|
||||
// 1403, met again once the hand rebuild no longer resumed the plan first).
|
||||
func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) {
|
||||
since := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}}
|
||||
waiting := busWaitingOnRetry(p)
|
||||
_, said := retryWords(p, []string{"a"}, nil, waiting)
|
||||
if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") {
|
||||
t.Fatalf("a retry left for the bus is said as %q", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own
|
||||
// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1).
|
||||
func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asksRecorded(t)
|
||||
planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
ended := before.Add(time.Minute)
|
||||
failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Delivery: &inventory.PlanDelivery{},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: "killed by hand"}},
|
||||
Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
// The controller's seat answers `plans` as it does: by running the verb.
|
||||
sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) {
|
||||
var args map[string]string
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
answer := link.Answer{Result: json.RawMessage(`"ok"`)}
|
||||
if len(args) != 1 || args["retry"] == "" {
|
||||
answer = link.Answer{Error: "the retry was asked with more than the plan"}
|
||||
} else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil {
|
||||
answer = link.Answer{Error: err.Error()}
|
||||
}
|
||||
body, _ := json.Marshal(answer)
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
// What S23 raises for it, read from the store as the watchdog reads it.
|
||||
raised := func(context.Context) ([]conditions.Condition, error) {
|
||||
facts, err := gatherFailedPlans(ctx, open.inventory, time.Now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) {
|
||||
out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
now := time.Now()
|
||||
store := memAskedStore{}
|
||||
var sent []published
|
||||
var acts []link.HandAct
|
||||
a := &asker{open: raised, store: store,
|
||||
silence: func(context.Context, string, time.Duration, string, string) error { return nil },
|
||||
publish: func(_ context.Context, subject string, body []byte, id string) error {
|
||||
sent = append(sent, published{subject, id, body})
|
||||
return nil
|
||||
},
|
||||
call: callAction(conn),
|
||||
record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil },
|
||||
now: func() time.Time { return now },
|
||||
logf: t.Logf}
|
||||
cs, _ := raised(ctx)
|
||||
if len(cs) != 1 || len(cs[0].Actions) != 1 {
|
||||
t.Fatalf("the failed plan raised %+v", cs)
|
||||
}
|
||||
if err := a.reconcile(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w asks.Warrant
|
||||
for _, r := range store {
|
||||
for _, o := range r.Ask.Options {
|
||||
if o.Label == "Retry" {
|
||||
w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen,
|
||||
Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now,
|
||||
AskDigest: r.Ask.Digest(),
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}}
|
||||
}
|
||||
}
|
||||
}
|
||||
if w.Ask == "" {
|
||||
t.Fatalf("no ask offers Retry: %+v", store)
|
||||
}
|
||||
body, _ := json.Marshal(w)
|
||||
if err := a.Decided(ctx, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.Decided(ctx, body); err != nil { // heard again: nothing more
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := p.Modules["a"]
|
||||
if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" {
|
||||
t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m)
|
||||
}
|
||||
if got := store[w.Ask]; got.Acted != "done" {
|
||||
t.Fatalf("the ask ended %q", got.Acted)
|
||||
}
|
||||
if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) {
|
||||
t.Fatalf("the warrant was recorded as %+v", acts)
|
||||
}
|
||||
if cs, _ = raised(ctx); len(cs) != 0 {
|
||||
t.Fatalf("the plan-failed condition did not end: %+v", cs)
|
||||
}
|
||||
}
|
||||
@@ -125,15 +125,8 @@ func newAskerRig(t *testing.T) *askerRig {
|
||||
}
|
||||
|
||||
func heldCondition() conditions.Condition {
|
||||
release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}}
|
||||
stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}}
|
||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096", Kind: "delivery-stalled",
|
||||
Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Headline: "Delivery of hq held for 36 hours",
|
||||
Explanation: "A delivery of hq has been held for 36 hours, past its limit.",
|
||||
Needs: "release it, or stop it, " + FromMeshMCPServer, Actions: []conditions.Action{release, stop}}
|
||||
o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s",
|
||||
Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0]
|
||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
}
|
||||
@@ -306,6 +299,35 @@ func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
|
||||
plan := "plan-1791454185265004861"
|
||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
||||
for _, tc := range []struct {
|
||||
c conditions.Condition
|
||||
label string
|
||||
want string
|
||||
}{
|
||||
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
|
||||
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
|
||||
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
|
||||
} {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{tc.c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
|
||||
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
|
||||
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.open = []conditions.Condition{unitsCondition()}
|
||||
@@ -1004,3 +1026,215 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) {
|
||||
t.Errorf("held %v (%q, %v) with nothing to read", held, why, err)
|
||||
}
|
||||
}
|
||||
|
||||
// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it.
|
||||
func failedPlanCondition(plan string) conditions.Condition {
|
||||
p := aFailedPlan(plan, nil)
|
||||
f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC))
|
||||
f.failed = []failedPlanFacts{failedFactsFor(p)}
|
||||
o := watchPlansFailed(f)[0]
|
||||
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
||||
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
||||
}
|
||||
|
||||
// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again".
|
||||
func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
r.open = []conditions.Condition{c}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := r.asksSent(t)
|
||||
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
||||
t.Fatalf("asked %+v", sent)
|
||||
}
|
||||
o := sent[0].Options[0]
|
||||
if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" {
|
||||
t.Errorf("the option is %+v", o)
|
||||
}
|
||||
if err := sent[0].Check(r.now); err != nil {
|
||||
t.Errorf("the ask is refused as made: %v", err)
|
||||
}
|
||||
if strings.Contains(sent[0].Explanation, "mesh MCP server") {
|
||||
t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1315/AC2: the warrant calls plans with retry=<plan> and nothing else, once, and is recorded as the hand act;
|
||||
// the same warrant heard again does nothing.
|
||||
func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition(plan)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, c.Key, "Retry")
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w)
|
||||
if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want {
|
||||
t.Fatalf("called %v, want exactly %s", r.called, want)
|
||||
}
|
||||
if len(r.acts) != 1 {
|
||||
t.Fatalf("hand-acts %+v", r.acts)
|
||||
}
|
||||
act := r.acts[0]
|
||||
if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan ||
|
||||
act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer {
|
||||
t.Errorf("the hand-act is %+v", act)
|
||||
}
|
||||
if got := r.store[w.Ask]; got.Acted != "done" {
|
||||
t.Errorf("kept %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is
|
||||
// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before.
|
||||
func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
|
||||
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
|
||||
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
c conditions.Condition
|
||||
label string
|
||||
}{
|
||||
{"retry", failedPlanCondition(plan), "Retry"},
|
||||
{"stop", waiting, "Stop"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"}
|
||||
r.open = []conditions.Condition{tc.c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
w := r.warrantFor(t, tc.c.Key, tc.label)
|
||||
answerWith(t, r, w)
|
||||
answerWith(t, r, w) // heard again: told once
|
||||
if len(r.told) != 1 {
|
||||
t.Fatalf("told %+v", r.told)
|
||||
}
|
||||
got := r.told[0]
|
||||
if got.headline != tc.c.Headline {
|
||||
t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline)
|
||||
}
|
||||
said := strings.Join(got.lines, " ")
|
||||
if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) {
|
||||
t.Errorf("the lines do not carry the answer and the refusal's words: %q", said)
|
||||
}
|
||||
if why, ok := conditions.Plain(said); !ok {
|
||||
t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said)
|
||||
}
|
||||
if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") {
|
||||
t.Errorf("the failure is not recorded as before: %+v", r.acts)
|
||||
}
|
||||
if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") {
|
||||
t.Errorf("kept %+v", kept)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here.
|
||||
func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
if len(r.told) != 0 {
|
||||
t.Fatalf("a retry that was done told %+v", r.told)
|
||||
}
|
||||
|
||||
r = newAskerRig(t)
|
||||
r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"}
|
||||
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
|
||||
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
|
||||
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
|
||||
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
|
||||
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
|
||||
r.open = []conditions.Condition{module}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, module.Key, "Restart"))
|
||||
if len(r.told) != 0 || len(r.acts) != 1 {
|
||||
t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts)
|
||||
}
|
||||
}
|
||||
|
||||
// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was
|
||||
// changed"; and the sentence a refusal falls back to fits the answer given.
|
||||
func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
r := newAskerRig(t)
|
||||
r.refuses = errors.New("mesh-controller did not answer plans within 30s")
|
||||
c := failedPlanCondition(plan)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
if len(r.told) != 1 {
|
||||
t.Fatalf("told %+v", r.told)
|
||||
}
|
||||
said := strings.Join(r.told[0].lines, " ")
|
||||
if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") {
|
||||
t.Errorf("a call that may have run is told as %q", said)
|
||||
}
|
||||
if why, ok := conditions.Plain(said); !ok {
|
||||
t.Errorf("not plain: %s: %q", why, said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) {
|
||||
plan := "plan-1792433168676870614"
|
||||
for _, tc := range []struct {
|
||||
act conditions.Action
|
||||
want string
|
||||
}{
|
||||
{retryAction(plan), "a retry would not run now"},
|
||||
{conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"},
|
||||
{conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"},
|
||||
} {
|
||||
// Words the operator's channel withholds: a hash and a dotted name.
|
||||
got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act)
|
||||
if !strings.Contains(got, tc.want) {
|
||||
t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want)
|
||||
}
|
||||
if _, ok := conditions.Plain(got); !ok {
|
||||
t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the
|
||||
// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not.
|
||||
func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) {
|
||||
r := newAskerRig(t)
|
||||
c := failedPlanCondition("plan-1792433168676870614")
|
||||
c.Raised = r.now.Add(-time.Minute)
|
||||
r.open = []conditions.Condition{c}
|
||||
_ = r.a.reconcile(context.Background())
|
||||
answerWith(t, r, r.warrantFor(t, c.Key, "Retry"))
|
||||
// Still open right after the answer: not asked again at once.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
_ = r.a.reconcile(context.Background())
|
||||
if n := len(r.asksSent(t)); n != 1 {
|
||||
t.Fatalf("asked again while the same condition stayed open: %d", n)
|
||||
}
|
||||
// The retry ran, the plan left failed and the condition ended.
|
||||
r.now = r.now.Add(time.Minute)
|
||||
r.open = nil
|
||||
_ = r.a.reconcile(context.Background())
|
||||
// It fails again, ten minutes after the answer: a condition raised after the answered ask ended.
|
||||
r.now = r.now.Add(8 * time.Minute)
|
||||
again := failedPlanCondition("plan-1792433168676870614")
|
||||
again.Raised = r.now
|
||||
r.open = []conditions.Condition{again}
|
||||
if err := r.a.reconcile(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := len(r.asksSent(t)); n != 2 {
|
||||
t.Fatalf("the second failure was asked %d times in all, want 2", n)
|
||||
}
|
||||
if len(r.called) != 1 {
|
||||
t.Fatalf("something retried without an answer: %v", r.called)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
|
||||
// can be compared about a secret that does not exist (status), and nothing here string-matches.
|
||||
_, _, err := composeQuestion(ctx, open, "laptop", compose.Reading, false)
|
||||
var notMade *catalogue.NotMadeError
|
||||
if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" {
|
||||
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
|
||||
}
|
||||
// Foreseen, it composes, names what the push makes, and made nothing.
|
||||
foreseen, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false)
|
||||
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
|
||||
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
|
||||
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
|
||||
}
|
||||
|
||||
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a machine waiting for a push raised %+v", got)
|
||||
}
|
||||
|
||||
// Past the bound: a warning, not urgent, saying what the push will make.
|
||||
before := awaitingPushBound
|
||||
awaitingPushBound = -time.Minute
|
||||
t.Cleanup(func() { awaitingPushBound = before })
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
|
||||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
|
||||
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
|
||||
}
|
||||
|
||||
// Pushed: the secret is made and D1 says nothing.
|
||||
pushedBy(t, open, "laptop")
|
||||
if got := d1(t, open); len(got) != 0 {
|
||||
t.Fatalf("a pushed machine still raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
|
||||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
|
||||
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
|
||||
}
|
||||
|
||||
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
|
||||
// uncomposable for that — the stand-in hides nothing.
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
_, _ = assign(ctx, open, "anchor", "rival-one")
|
||||
_, _ = assign(ctx, open, "anchor", "rival-two")
|
||||
keys := map[string]conditions.Severity{}
|
||||
for _, o := range d1(t, open) {
|
||||
keys[o.Key()] = o.Severity
|
||||
}
|
||||
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
|
||||
t.Fatalf("a real failure beside a waiting secret: %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := d1(t, open)
|
||||
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
|
||||
t.Fatalf("a given secret under an old key: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
sent := time.Now()
|
||||
since, err := open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.After(sent) {
|
||||
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since, err = open.inventory.AwaitingSince(ctx, "laptop")
|
||||
if err != nil || since.Before(sent.Add(-time.Second)) {
|
||||
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
|
||||
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
|
||||
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sends, err := readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("a holder assigned and not pushed is expected to answer")
|
||||
}
|
||||
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
|
||||
t.Fatal("a module the machine was sent long ago is not expected to answer")
|
||||
}
|
||||
pushedBy(t, open, "laptop")
|
||||
sends, err = readDeliveries(ctx, open.inventory)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
|
||||
}
|
||||
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
|
||||
t.Fatal("pushed past the grace, a holder is not expected")
|
||||
}
|
||||
if _, ok := sends["laptop"].carried["keeper"]; !ok {
|
||||
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
|
||||
}
|
||||
pushedAndApplied(t, open, "laptop")
|
||||
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
|
||||
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
|
||||
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
|
||||
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
|
||||
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
|
||||
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
|
||||
|
||||
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
|
||||
func aKeeper() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
|
||||
// are made — without a bus to carry it.
|
||||
func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
plan, settings, err := resolveNode(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
|
||||
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
|
||||
builds := declared.Builds
|
||||
if builds == nil {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builds = map[string]string{}
|
||||
for _, m := range plan.Modules {
|
||||
builds[m.Module] = current[m.Module].Commit
|
||||
}
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
|
||||
func pushedAndApplied(t *testing.T, open *stores, node string) {
|
||||
t.Helper()
|
||||
digest := pushedBy(t, open, node)
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// d1 runs D1 once.
|
||||
func d1(t *testing.T, open *stores) []conditions.Observation {
|
||||
t.Helper()
|
||||
got, err := probeDeclarations(t.Context(), &doctor{open: open})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
|
||||
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
|
||||
|
||||
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
|
||||
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
|
||||
|
||||
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
|
||||
// refused on it, so D1 is too.
|
||||
|
||||
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
|
||||
|
||||
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
|
||||
// it and had time to report — never between assign and push.
|
||||
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-time.Hour)
|
||||
cases := []struct {
|
||||
name string
|
||||
send lastSend
|
||||
want bool
|
||||
}{
|
||||
{"never sent", lastSend{}, false},
|
||||
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
|
||||
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
|
||||
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
|
||||
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.send.settled("keeper", now); got != c.want {
|
||||
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
||||
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
||||
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
||||
|
||||
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
||||
// is away once presence says so). A machine
|
||||
// never heard from has not joined and is not away.
|
||||
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
if m.silent(now) {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
||||
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
||||
// before: a machine is not passed on a reading that was not made.
|
||||
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
machines := make([]machineFacts, len(nodes))
|
||||
for i, n := range nodes {
|
||||
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
||||
}
|
||||
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return awayOf(machines, now), nil
|
||||
}
|
||||
|
||||
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
||||
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
||||
for _, n := range running {
|
||||
if away[n] && !slices.Contains(p.Away, n) {
|
||||
p.Away = append(p.Away, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(p.Away)
|
||||
}
|
||||
|
||||
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
||||
// those it did not pass as away.
|
||||
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
||||
if !slices.Contains(p.Away, n) {
|
||||
return false
|
||||
}
|
||||
_, sent := s.Rest[n]
|
||||
return !sent
|
||||
})
|
||||
}
|
||||
|
||||
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
||||
func awaySaying(p inventory.Plan) string {
|
||||
var said []string
|
||||
for _, n := range p.Away {
|
||||
said = append(said, n+" was away: sent when it is heard again")
|
||||
}
|
||||
return strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
||||
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
||||
away, err := judgeAway(ctx, open, now)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
||||
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
||||
}
|
||||
return away
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it
|
||||
// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard
|
||||
// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's
|
||||
// builds, so the send it gets when it is heard again carries them (AC3).
|
||||
func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
// laptop is away as every tier is sent; anchor is heard.
|
||||
wasAway := judgeAway
|
||||
judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true}, nil
|
||||
}
|
||||
t.Cleanup(func() { judgeAway = wasAway })
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = nil })
|
||||
sentHeard := make(chan *nats.Msg, 32)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
wasSaved := inventory.PlanSaved
|
||||
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||
|
||||
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||
var mu sync.Mutex
|
||||
var sends [][]string
|
||||
everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it
|
||||
// Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look
|
||||
// goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask
|
||||
// was where the walk paused).
|
||||
wholeKeys := openPlanKeys
|
||||
var holdB atomic.Bool
|
||||
holdB.Store(true)
|
||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
all, done, err := wholeKeys()
|
||||
return keysHolding{all, "b", &holdB, ""}, done, err
|
||||
}
|
||||
digests := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
sends = append(sends, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds := map[string]string{}
|
||||
for _, m := range []string{"a", "b"} {
|
||||
builds[m] = current[m].Commit
|
||||
}
|
||||
for _, node := range names {
|
||||
digests++
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
register := func(m, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record := func(id, m, commit string, at time.Time) {
|
||||
t.Helper()
|
||||
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
for _, m := range []string{"a", "b"} {
|
||||
record("build-"+m+"-1", m, "c1", old)
|
||||
register(m, "c1", old)
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||
register("a", "c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// A machine's report on what it was sent, heard as the controller hears it.
|
||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||
seq := int64(0)
|
||||
reports := func(node string) {
|
||||
t.Helper()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||
}
|
||||
seq++
|
||||
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||
"report_sequence": seq})
|
||||
var report link.Report
|
||||
if err := json.Unmarshal(body, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := listener.Heard(ctx, report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
plan := func() inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := inv.PlanByID(ctx, "plan-1160")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
built := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||
"b": {},
|
||||
}}
|
||||
first, stopFirst := context.WithCancel(ctx)
|
||||
defer stopFirst()
|
||||
waitFirst := startRolloutEvents(first, open, js)
|
||||
// Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338).
|
||||
t.Cleanup(func() { stopFirst(); waitFirst() })
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "tier 0 sent to anchor alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor"
|
||||
})
|
||||
reports("anchor")
|
||||
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
||||
var moved string
|
||||
eventually(t, "tier 1 reached on anchor's report alone", func() bool {
|
||||
if q := plan(); q.Tier == 1 {
|
||||
moved = q.Note
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
|
||||
t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
|
||||
}
|
||||
// Tier 1's module is taken from its key once that is built, which the walk's look does by itself.
|
||||
holdB.Store(false)
|
||||
askToLook()
|
||||
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
||||
})
|
||||
// 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again.
|
||||
was, known, err := inv.SentBuilds(ctx, "laptop")
|
||||
if err != nil || !known {
|
||||
t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if was["a"] == current["a"].Commit || was["a"] != "c1" {
|
||||
t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit)
|
||||
}
|
||||
if got := plan().Away; strings.Join(got, ",") != "laptop" {
|
||||
t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
|
||||
}
|
||||
}
|
||||
|
||||
// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of
|
||||
// the fingerprint fp when that is given.
|
||||
type keysHolding struct {
|
||||
planKeyReads
|
||||
module string
|
||||
held *atomic.Bool
|
||||
fp string
|
||||
}
|
||||
|
||||
func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) {
|
||||
if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) {
|
||||
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
|
||||
}
|
||||
return k.planKeyReads.Build(ctx, module, fp)
|
||||
}
|
||||
|
||||
// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared,
|
||||
// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of
|
||||
// the test, and the flag is set until the test lowers it.
|
||||
func holdKeys(t *testing.T, modules ...string) *atomic.Bool {
|
||||
t.Helper()
|
||||
held := &atomic.Bool{}
|
||||
held.Store(true)
|
||||
was := openPlanKeys
|
||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
all, done, err := was()
|
||||
var keys planKeyReads = all
|
||||
for _, m := range modules {
|
||||
keys = keysHolding{keys, m, held, ""}
|
||||
}
|
||||
return keys, done, err
|
||||
}
|
||||
t.Cleanup(func() { openPlanKeys = was })
|
||||
return held
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
||||
|
||||
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
||||
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
machines := []machineFacts{
|
||||
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
||||
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
||||
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
||||
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
||||
{name: "new"},
|
||||
}
|
||||
got := awayOf(machines, now)
|
||||
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
||||
"new": false} {
|
||||
if got[node] != want {
|
||||
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
||||
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
||||
p := &inventory.Plan{}
|
||||
away := map[string]bool{"g14": true, "ace": true}
|
||||
rememberAway(p, []string{"anchor", "g14"}, away)
|
||||
rememberAway(p, []string{"g14", "ace"}, away)
|
||||
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
||||
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC2: the plan says it, where its other passed machines are said.
|
||||
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
||||
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
||||
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
||||
t.Fatalf("1161/AC2: the plan says %q", say)
|
||||
}
|
||||
if got := awaySaying(inventory.Plan{}); got != "" {
|
||||
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
||||
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
||||
p := inventory.Plan{Away: []string{"g14"}}
|
||||
none := &inventory.PlanModule{}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
||||
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
||||
}
|
||||
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
||||
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and
|
||||
// answers `backed-up` with the last night it is told (novox/hq ADR 0404).
|
||||
type aBackupSeatOnTheBus struct {
|
||||
mu sync.Mutex
|
||||
nows []string
|
||||
night map[string]any // module → lastNight
|
||||
}
|
||||
|
||||
func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) {
|
||||
t.Helper()
|
||||
reply := func(m *nats.Msg, result any) {
|
||||
body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node})
|
||||
_ = m.Respond(body)
|
||||
}
|
||||
now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) {
|
||||
var a struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
f.mu.Lock()
|
||||
f.nows = append(f.nows, a.Module)
|
||||
f.mu.Unlock()
|
||||
reply(m, map[string]any{"started": a.Module})
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []map[string]any
|
||||
for module, n := range f.night {
|
||||
out = append(out, map[string]any{"module": module, "lastNight": n})
|
||||
}
|
||||
reply(m, out)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() })
|
||||
if err := conn.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}}
|
||||
}
|
||||
|
||||
func (f *aBackupSeatOnTheBus) asks() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.nows)
|
||||
}
|
||||
|
||||
// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits
|
||||
// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the
|
||||
// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the
|
||||
// backups again, and the plan goes on. One bus and one store for the scenarios.
|
||||
func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
|
||||
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
|
||||
`"resources":[{"id":"files","type":"directory"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := assign(ctx, open, node, "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, node, "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}}
|
||||
for node, s := range seats {
|
||||
s.serve(t, js.Conn(), node)
|
||||
}
|
||||
// Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup.
|
||||
wasWithin, wasEvery := backupsWithin, backupsPollEvery
|
||||
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
|
||||
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
|
||||
|
||||
newPlan := func(id string) {
|
||||
t.Helper()
|
||||
p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
read := func(id string) inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// 1. Both machines are asked; while one has not answered, no build is asked.
|
||||
newPlan("plan-one")
|
||||
advancePlans(ctx, open)
|
||||
if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 {
|
||||
t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked)
|
||||
}
|
||||
seats["anchor"].answers("notes", true, "")
|
||||
advancePlans(ctx, open)
|
||||
if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending {
|
||||
t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups)
|
||||
}
|
||||
seats["laptop"].answers("notes", true, "")
|
||||
advancePlans(ctx, open)
|
||||
if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone {
|
||||
t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups)
|
||||
}
|
||||
finish := read("plan-one")
|
||||
finish.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &finish); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised.
|
||||
seats["anchor"].night, seats["laptop"].night = nil, nil
|
||||
newPlan("plan-two")
|
||||
advancePlans(ctx, open)
|
||||
seats["anchor"].answers("notes", true, "")
|
||||
seats["laptop"].answers("notes", false, "no space left on device")
|
||||
advancePlans(ctx, open)
|
||||
held := read("plan-two")
|
||||
if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 {
|
||||
t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked)
|
||||
}
|
||||
var failed *inventory.PlanBackup
|
||||
for i := range held.Backups.Asked {
|
||||
if held.Backups.Asked[i].State == inventory.BackupFailed {
|
||||
failed = &held.Backups.Asked[i]
|
||||
}
|
||||
}
|
||||
if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" ||
|
||||
strings.Contains(held.Note, "tried again") {
|
||||
t.Fatalf("the failed backup: %+v note=%q", failed, held.Note)
|
||||
}
|
||||
advancePlans(ctx, open) // not retried on the next look
|
||||
if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 ||
|
||||
seats["laptop"].asks() != 2 {
|
||||
t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks())
|
||||
}
|
||||
said, err := probeBackupHeld(ctx, &doctor{open: open})
|
||||
if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") {
|
||||
t.Fatalf("the condition: %+v %v", said, err)
|
||||
}
|
||||
|
||||
// 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on.
|
||||
seats["laptop"].answers("notes", false, "no space left on device")
|
||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil {
|
||||
t.Fatal("a plan with no failed backup was asked again")
|
||||
}
|
||||
seats["laptop"].night = nil
|
||||
advancePlans(ctx, open)
|
||||
if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 {
|
||||
t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks())
|
||||
}
|
||||
seats["laptop"].answers("notes", true, "")
|
||||
advancePlans(ctx, open)
|
||||
// One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key
|
||||
// build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes
|
||||
// it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key.
|
||||
if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 ||
|
||||
got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" {
|
||||
t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked)
|
||||
}
|
||||
if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 {
|
||||
t.Fatalf("the condition stays: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1086: a plan cut from a merge asked its first tier at the cut, so a module keeping data was sent without the
|
||||
// backup the plan promised. The cut now takes the backups first (askFirstTierAtCut, batches.go), and the walk's own
|
||||
// step asks the tier once they are done.
|
||||
func TestAPlanCutFromAMergeOfABackedUpModuleAsksItsBackupAndNotItsTierUntilItIsDone(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
everyHeadWaits(t)
|
||||
asked := asksRecorded(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` +
|
||||
`{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` +
|
||||
`"resources":[{"id":"files","type":"directory"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/notes", Seat: "git", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []string{"keeper", "notes"} {
|
||||
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
seat := &aBackupSeatOnTheBus{}
|
||||
seat.serve(t, js.Conn(), "anchor")
|
||||
wasWithin, wasEvery := backupsWithin, backupsPollEvery
|
||||
backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond
|
||||
t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery })
|
||||
|
||||
// 1. A merge of notes is heard and its window closes: the batch is cut through cutBatch.
|
||||
hear(t, open, repoMerge("notes", "c-notes", t0), t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walk(s)", len(ws))
|
||||
}
|
||||
// 2. Right after the cut no tier is asked, the record has anchor/notes pending, the seat heard one `now`.
|
||||
cut, err := inv.PlanByID(ctx, ws[0].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 || askedInTier(cut) {
|
||||
t.Fatalf("the first tier was asked at the cut: %v %+v", *asked, cut.Modules)
|
||||
}
|
||||
if cut.Backups == nil || cut.Backups.State != inventory.BackupPending || len(cut.Backups.Asked) != 1 ||
|
||||
cut.Backups.Asked[0].Machine != "anchor" || cut.Backups.Asked[0].Module != "notes" {
|
||||
t.Fatalf("the backups of the cut: %+v", cut.Backups)
|
||||
}
|
||||
if seat.asks() != 1 {
|
||||
t.Fatalf("the seat heard %d `now`", seat.asks())
|
||||
}
|
||||
// 3. The backup answers done: the walk's own step asks the first tier, once, and asks no second backup.
|
||||
seat.answers("notes", true, "")
|
||||
advancePlans(ctx, open)
|
||||
advancePlans(ctx, open)
|
||||
after, err := inv.PlanByID(ctx, ws[0].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 1 || after.Backups.State != inventory.BackupDone || seat.asks() != 1 {
|
||||
t.Fatalf("builds=%v backups=%+v asks=%d", *asked, after.Backups, seat.asks())
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,727 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func windowed(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
settings := t.TempDir() + "/merge-window.json"
|
||||
if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, settings)
|
||||
noGroups(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []struct{ module, repository, path string }{
|
||||
{"app", "novox/mesh-catalog", "modules/app"},
|
||||
{"notes", "novox/mesh-catalog", "modules/notes"},
|
||||
{"one", "novox/one", ""},
|
||||
{"two", "novox/two", ""},
|
||||
{"three", "novox/three", ""},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"},
|
||||
inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) {
|
||||
t.Helper()
|
||||
if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func cutAt(t *testing.T, open *stores, now time.Time) {
|
||||
t.Helper()
|
||||
if err := cutBatchesHeld(t.Context(), open, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
claude := catalogueMerge("553b7191claude", "app", t0)
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, claude, t0.Add(time.Second))
|
||||
hear(t, open, dunst, t0.Add(18*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+89*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatalf("cut before the window closed: %+v", ws)
|
||||
}
|
||||
cutAt(t, open, t0.Add(18*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit {
|
||||
t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := w.Modules[m]; !in {
|
||||
t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules)
|
||||
}
|
||||
}
|
||||
want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit},
|
||||
{Repository: "novox/mesh-catalog", Commit: dunst.Commit}}
|
||||
if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) {
|
||||
t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want)
|
||||
}
|
||||
// The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at
|
||||
// which commit: the one the branch holds, which is the later merge the walk carries.
|
||||
for _, m := range w.Tiers[w.Tier] {
|
||||
if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit {
|
||||
t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s)
|
||||
}
|
||||
}
|
||||
// Heard again, as the bus may hand it over twice: never a second merge, nor a second walk.
|
||||
hear(t, open, claude, t0.Add(5*time.Minute))
|
||||
if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/three")
|
||||
asksRecorded(t)
|
||||
for i, r := range []string{"one", "two", "three"} {
|
||||
hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"),
|
||||
t0.Add(time.Duration(i)*20*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(41*time.Second+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
w := ws[0]
|
||||
if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges)
|
||||
}
|
||||
for _, r := range []string{"one", "two", "three"} {
|
||||
if w.CommitOf("novox/"+r) != "c-"+r {
|
||||
t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r))
|
||||
}
|
||||
if _, in := w.Modules[r]; !in {
|
||||
t.Fatalf("the walk does not build %s: %v", r, w.Modules)
|
||||
}
|
||||
}
|
||||
// Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build).
|
||||
for _, r := range w.Tiers[w.Tier] {
|
||||
if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r {
|
||||
t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second))
|
||||
cutAt(t, open, t0.Add(100*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 0 {
|
||||
t.Fatal("the window closed 90 seconds after its first merge, not after its last")
|
||||
}
|
||||
cutAt(t, open, t0.Add(170*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 {
|
||||
t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws)
|
||||
}
|
||||
|
||||
busy := windowed(t)
|
||||
groupsWaiting(t, "feat/both", "feat/busy")
|
||||
asksWithPaths(t)
|
||||
var last time.Time
|
||||
for i := 0; i < 12; i++ {
|
||||
last = t0.Add(time.Duration(i) * time.Minute)
|
||||
repo := []string{"one", "two", "three"}[i%3]
|
||||
hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last)
|
||||
if ws, _ := walks(t, busy); len(ws) > 0 {
|
||||
if i != 10 {
|
||||
t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
ws, _ := walks(t, busy)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAssemblingBatchIsShown(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/553b7191claude")
|
||||
asksWithPaths(t)
|
||||
var said []inventory.Plan
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) }
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
now := time.Now().UTC()
|
||||
claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second))
|
||||
claude.Number, claude.Title = 174, "claude-code: an agent proposes a section"
|
||||
dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second))
|
||||
dunst.Number, dunst.Title = 175, "dunst: the font the operator chose"
|
||||
hear(t, open, claude, now.Add(-19*time.Second))
|
||||
hear(t, open, dunst, now.Add(-time.Second))
|
||||
hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now)
|
||||
out := captured(t, func() error { return plansCommand(t.Context(), nil) })
|
||||
lines := strings.Split(out, "\n")
|
||||
first := lines[0]
|
||||
for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " +
|
||||
"(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Fatalf("plans' first line %q does not say %q", first, want)
|
||||
}
|
||||
}
|
||||
// Under it, one line per repository: the pull request, what it answers, what it moves.
|
||||
if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+
|
||||
"#174 claude-code: an agent proposes a section) · app, notes" ||
|
||||
strings.TrimSpace(lines[2]) != "one a6bc0931 · one" {
|
||||
t.Fatalf("the grouped list reads %q", lines[1:4])
|
||||
}
|
||||
if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil ||
|
||||
len(said[len(said)-1].Delivery.Merges) != 3 {
|
||||
t.Fatalf("the batch was not said as assembling with its merges: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
func captured(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := os.Stdout
|
||||
os.Stdout = w
|
||||
runErr := run()
|
||||
os.Stdout = was
|
||||
_ = w.Close()
|
||||
var b bytes.Buffer
|
||||
_, _ = io.Copy(&b, r)
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first batch was not cut: %+v", ws)
|
||||
}
|
||||
first := ws[0]
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(5*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID {
|
||||
t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the next merge", got.State)
|
||||
}
|
||||
if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) {
|
||||
t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute)))
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(t.Context(), &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" {
|
||||
t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || !ws[0].Waiting() {
|
||||
t.Fatalf("the walk does not wait for its word: %+v", ws)
|
||||
}
|
||||
waiting := ws[0]
|
||||
hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute))
|
||||
cutAt(t, open, t0.Add(2*time.Minute+90*time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(bs) != 0 || len(ws) != 2 {
|
||||
t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
folded, err := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer := ws[0]
|
||||
if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID {
|
||||
t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy)
|
||||
}
|
||||
if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" {
|
||||
t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges)
|
||||
}
|
||||
if _, in := newer.Modules["one"]; !in {
|
||||
t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second))
|
||||
hear(t, open, later, t0.Add(18*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Moves only notes, which the done walk built from the branch after it: answered there, at once.
|
||||
hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 ||
|
||||
!sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early",
|
||||
Carried: later.Commit}) {
|
||||
t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
// Moves app, which that walk never built: the next batch walks it.
|
||||
hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" {
|
||||
t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
// Made before the cut marks their modules seen, as merges are: walked again, they are news all the same.
|
||||
oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute))
|
||||
middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second))
|
||||
newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second))
|
||||
for i, m := range []link.SourceMoved{oldest, middle, newest} {
|
||||
hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second))
|
||||
}
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
failed := ws[0]
|
||||
failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0"
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 {
|
||||
t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges)
|
||||
}
|
||||
ws, _ = walks(t, open)
|
||||
alone := ws[0]
|
||||
if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 ||
|
||||
!sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) {
|
||||
t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone)
|
||||
}
|
||||
// The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the
|
||||
// module is taken from its key at the commit the merge was walked on.
|
||||
if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit {
|
||||
t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules)
|
||||
}
|
||||
// Retried, the failed walk would name merges the search answers now.
|
||||
if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") {
|
||||
t.Fatalf("a failed walk whose merges are searched was retried: %v", err)
|
||||
}
|
||||
alone.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &alone); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(4*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if ws[0].ID != alone.ID || len(bs) != 0 {
|
||||
t.Fatalf("the search went on after a merge was delivered: %+v", ws[0])
|
||||
}
|
||||
done, _ := open.inventory.PlanByID(ctx, alone.ID)
|
||||
if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: oldest.Commit, Carried: middle.Commit}) {
|
||||
t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges)
|
||||
}
|
||||
// A stopped walk starts no search: a person ended it.
|
||||
hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute))
|
||||
hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second))
|
||||
cutAt(t, open, t0.Add(8*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(9*time.Minute))
|
||||
if again, _ := walks(t, open); again[0].ID != ws[0].ID {
|
||||
t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) {
|
||||
for _, grouped := range []bool{true, false} {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"mesh-host"} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
host := repoMerge("mesh-host", "h1", t0)
|
||||
ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second))
|
||||
host.Head, ctl.Head = "feat/together", "feat/together"
|
||||
if !grouped {
|
||||
ctl.Head = "feat/alone"
|
||||
}
|
||||
hear(t, open, ctl, t0.Add(2*time.Second))
|
||||
hear(t, open, host, t0.Add(3*time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walks", len(ws))
|
||||
}
|
||||
tiers := ws[0].Tiers
|
||||
if grouped {
|
||||
if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) ||
|
||||
!slices.Equal(tiers[1], []string{"mesh-controller"}) {
|
||||
t.Fatalf("the group's order is not the walk's tiers: %v", tiers)
|
||||
}
|
||||
} else if len(tiers) != 1 {
|
||||
t.Fatalf("two merges of no group were ordered: %v", tiers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABatchSurvivesARestart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "feat/both")
|
||||
asksWithPaths(t)
|
||||
hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0)
|
||||
hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second))
|
||||
|
||||
// A new controller: nothing of the first one's but the store.
|
||||
again, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(again.Close)
|
||||
hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again
|
||||
cutAt(t, again, t0.Add(119*time.Second))
|
||||
if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs)
|
||||
}
|
||||
cutAt(t, again, t0.Add(120*time.Second))
|
||||
ws, bs := walks(t, again)
|
||||
if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
c2 := repoMerge("one", "c2", t0.Add(-50*time.Second))
|
||||
hear(t, open, c2, t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
w := ws[0]
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, w.ID)
|
||||
if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one",
|
||||
Commit: "c1", Carried: "c2"}) {
|
||||
t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
|
||||
// A walk of two that built only two: a late merge of one is not its to name.
|
||||
hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute))
|
||||
cutAt(t, open, t0.Add(18*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
w = ws[0]
|
||||
delete(w.Modules, "two")
|
||||
w.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute))
|
||||
if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" {
|
||||
t.Fatalf("the late merge did not join the next batch: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asked := asksWithPaths(t)
|
||||
// The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with
|
||||
// the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other.
|
||||
wholeKeys, held := openPlanKeys, &atomic.Bool{}
|
||||
held.Store(true)
|
||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
all, done, err := wholeKeys()
|
||||
return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err
|
||||
}
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
started, _ := walks(t, open)
|
||||
waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatalf("the word was refused beside %s: %v", started[0].ID, err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ := open.inventory.PlanByID(ctx, waiting.ID)
|
||||
if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) {
|
||||
t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note)
|
||||
}
|
||||
// Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3.
|
||||
later := time.Now().Add(time.Hour)
|
||||
if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") {
|
||||
t.Fatalf("a deferred walk reads %q", line)
|
||||
}
|
||||
facts, _, err := gatherPlans(ctx, open.inventory, later, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range facts {
|
||||
if f.id == got.ID {
|
||||
t.Fatalf("a deferred walk is watched as a tier running late: %+v", f)
|
||||
}
|
||||
}
|
||||
// Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale.
|
||||
done, err := open.inventory.PlanByID(ctx, started[0].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before+1 {
|
||||
t.Fatalf("the walk did not start once the started one ended: asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0)
|
||||
hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second))
|
||||
hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
var ownBatch, catalogueBatch inventory.Plan
|
||||
for _, b := range bs {
|
||||
if b.OwnPath() {
|
||||
ownBatch = b
|
||||
} else {
|
||||
catalogueBatch = b
|
||||
}
|
||||
}
|
||||
if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 ||
|
||||
len(ownBatch.Delivery.Merges) != 1 {
|
||||
t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs)
|
||||
}
|
||||
if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") {
|
||||
t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second)))
|
||||
}
|
||||
// Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it,
|
||||
// is cut when the controller's walk ended, and waits for its word with both merges.
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws)
|
||||
}
|
||||
ownWalk := ws[0]
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ownWalk.Modules[m]; in {
|
||||
t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m)
|
||||
}
|
||||
}
|
||||
if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() {
|
||||
t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs)
|
||||
}
|
||||
ownWalk.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, t0.Add(3*time.Minute))
|
||||
ws, bs = walks(t, open)
|
||||
if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 {
|
||||
t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs)
|
||||
}
|
||||
for _, m := range []string{"app", "notes"} {
|
||||
if _, in := ws[0].Modules[m]; !in {
|
||||
t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules)
|
||||
}
|
||||
}
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/app" || a[1] == "modules/notes" {
|
||||
t.Fatalf("a catalogue module was asked without the word: %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path
|
||||
// walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended.
|
||||
catalogueWalk := ws[0]
|
||||
// The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the
|
||||
// stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk.
|
||||
wholeKeys, held := openPlanKeys, &atomic.Bool{}
|
||||
held.Store(true)
|
||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
all, done, err := wholeKeys()
|
||||
return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err
|
||||
}
|
||||
hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute))
|
||||
cutAt(t, open, t0.Add(6*time.Minute))
|
||||
ws, _ = walks(t, open)
|
||||
kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID)
|
||||
if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() {
|
||||
t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws)
|
||||
}
|
||||
if _, in := ws[0].Modules["app"]; in {
|
||||
t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules)
|
||||
}
|
||||
if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := len(*asked)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != before {
|
||||
t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:])
|
||||
}
|
||||
// Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale.
|
||||
own2, err := open.inventory.PlanByID(ctx, ws[0].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
own2.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &own2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" {
|
||||
t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
for _, m := range []struct {
|
||||
module string
|
||||
claims []catalogue.Claim
|
||||
}{
|
||||
{"nats", nil},
|
||||
{"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}},
|
||||
} {
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano),
|
||||
Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"},
|
||||
ModuleDirsSaid: true}
|
||||
hear(t, open, mixed, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 || ws[0].Waiting() {
|
||||
t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws)
|
||||
}
|
||||
done := ws[0]
|
||||
done.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute))
|
||||
got, _ := open.inventory.PlanByID(ctx, done.ID)
|
||||
_, bs := walks(t, open)
|
||||
if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" {
|
||||
t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)",
|
||||
got.Delivery.Merges, bs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a
|
||||
// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge
|
||||
// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings.
|
||||
func init() {
|
||||
mergeWindowDefault = 0
|
||||
// No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change
|
||||
// (delivery_edges.go). A test that wants some replaces these (forgeHolding).
|
||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
}
|
||||
|
||||
// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its
|
||||
// settings file says them, and
|
||||
// modules built from the catalogue (app, notes) and from three repositories of their own.
|
||||
|
||||
// t0 is the moment a test's clock starts: merges are dated by it, and heard after it.
|
||||
var t0 = time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull
|
||||
// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group
|
||||
// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test
|
||||
// whose merges wait for the window as a group's says so (groupsWaiting).
|
||||
func catalogueMerge(commit, module string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"},
|
||||
ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true}
|
||||
}
|
||||
|
||||
// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge).
|
||||
func repoMerge(repo, commit string, made time.Time) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit,
|
||||
MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}}
|
||||
}
|
||||
|
||||
// noGroups stubs mesh-delivery's word on groups for a test: no head names a group.
|
||||
func noGroups(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil }
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named
|
||||
// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and
|
||||
// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds
|
||||
// its batch open however long it takes (ADR 0418 clause 3), is groupStates'.
|
||||
func groupsWaiting(t *testing.T, heads ...string) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) {
|
||||
if slices.Contains(heads, head) {
|
||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did
|
||||
// before novox/hq ADR 0304: mesh-delivery cannot say any head's group.
|
||||
func everyHeadWaits(t *testing.T) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) {
|
||||
return nil, false, errors.New("the test's mesh-delivery does not say")
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the
|
||||
// members merge.
|
||||
func groupStates(t *testing.T, head string, states map[string]string) {
|
||||
t.Helper()
|
||||
groupStatesFrom(t, head, states, true)
|
||||
}
|
||||
|
||||
// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the
|
||||
// composed check's: only the first holds the group's batch open for its last member.
|
||||
func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) {
|
||||
t.Helper()
|
||||
was := deliveryGroupOf
|
||||
deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) {
|
||||
if asked != head {
|
||||
return nil, false, nil
|
||||
}
|
||||
var members []groupMember
|
||||
for repository, state := range states {
|
||||
members = append(members, groupMember{Repository: repository, State: state, Composed: composed})
|
||||
}
|
||||
return members, true, nil
|
||||
}
|
||||
t.Cleanup(func() { deliveryGroupOf = was })
|
||||
}
|
||||
|
||||
// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group,
|
||||
// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share.
|
||||
func fromHead(m link.SourceMoved, head string) link.SourceMoved {
|
||||
m.Head = head
|
||||
return m
|
||||
}
|
||||
|
||||
// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is
|
||||
// related to a batch assembling (ADR 0304 decision 1).
|
||||
func lone(m link.SourceMoved) link.SourceMoved {
|
||||
m.Head = ""
|
||||
return m
|
||||
}
|
||||
|
||||
// hear is a merge heard at a moment.
|
||||
|
||||
// cutAt is the cutter looking at a moment.
|
||||
|
||||
// walks is every plan record that is a walk, newest first, and the batches not yet cut.
|
||||
func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) {
|
||||
t.Helper()
|
||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range recent {
|
||||
if p.Batch() {
|
||||
batches = append(batches, p)
|
||||
} else {
|
||||
walks = append(walks, p)
|
||||
}
|
||||
}
|
||||
return walks, batches
|
||||
}
|
||||
|
||||
// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the
|
||||
// later commit, which names both merges — the earlier carried by the later.
|
||||
|
||||
// Merges of three repositories in one window are one walk, one commit for each.
|
||||
|
||||
// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window
|
||||
// closes at most ten minutes after its first merge however busy.
|
||||
|
||||
// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest,
|
||||
// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too.
|
||||
|
||||
// captured is what a command printed.
|
||||
|
||||
// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is
|
||||
// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4).
|
||||
|
||||
// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it
|
||||
// names the walk that took it over.
|
||||
|
||||
// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it
|
||||
// moves — named at once on a walk done — and joins the next batch when it does not.
|
||||
|
||||
// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest
|
||||
// first; the first delivered answers the older ones, and the search stops.
|
||||
|
||||
// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head
|
||||
// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier.
|
||||
|
||||
// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the
|
||||
// store, a merge handed over again is not doubled, and the batch is cut when its window closes.
|
||||
|
||||
// The window's settings reach the controller in its settings file, read at every look; one that says no
|
||||
// duration, or no file, is the default.
|
||||
func TestTheWindowIsTheControllersSetting(t *testing.T) {
|
||||
file := t.TempDir() + "/merge-window.json"
|
||||
t.Setenv(mergeWindowFileVar, file)
|
||||
for _, c := range []struct {
|
||||
content string
|
||||
window, atMost time.Duration
|
||||
}{
|
||||
{`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute},
|
||||
{`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault},
|
||||
{`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault},
|
||||
} {
|
||||
if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w, m := mergeWindowOf(); w != c.window || m != c.atMost {
|
||||
t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost)
|
||||
}
|
||||
}
|
||||
t.Setenv(mergeWindowFileVar, file+".gone")
|
||||
if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault {
|
||||
t.Errorf("no file reads as %s and %s", w, m)
|
||||
}
|
||||
}
|
||||
|
||||
// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362).
|
||||
func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) {
|
||||
f := calm(t0)
|
||||
f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery",
|
||||
since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"},
|
||||
{Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}}
|
||||
got := watchWaits(f)
|
||||
if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") ||
|
||||
!strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") {
|
||||
t.Fatalf("the waiting walk does not name both merges: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since
|
||||
// the cut, in a repository of one module (review of this change); a walk that never built what it moves names
|
||||
// nothing, and the merge joins the next batch.
|
||||
|
||||
// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to
|
||||
// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before.
|
||||
|
||||
// A file a merge of the batch removed and a later one brought back is not removed; one removed last is.
|
||||
func TestARemovedFileIsTheLastMergesWord(t *testing.T) {
|
||||
ev := func(commit string, paths, removed []string) inventory.BatchedMerge {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed}
|
||||
b, _ := json.Marshal(m)
|
||||
return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b,
|
||||
Merged: t0.Add(time.Duration(len(commit)) * time.Second)}
|
||||
}
|
||||
got := combinedMerges([]inventory.BatchedMerge{
|
||||
ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}),
|
||||
ev("bb", []string{"x/module.json"}, nil),
|
||||
ev("ccc", []string{"z.go"}, nil)})
|
||||
if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) ||
|
||||
len(got[0].Paths) != 3 {
|
||||
t.Fatalf("combined as %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word
|
||||
// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on
|
||||
// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk
|
||||
// still waits for the word, so no catalogue delivery skips its turn.
|
||||
|
||||
// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and
|
||||
// title, the modules it moves and the machines running them, then the state words; a record naming no pull
|
||||
// request is named by its commit, and a title is cut at fifty runes.
|
||||
func TestAPlanLineNamesWhatItMoves(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1",
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute),
|
||||
Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}},
|
||||
Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}},
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog",
|
||||
Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}}
|
||||
running := func(module string) []string {
|
||||
if module == "messenger" || module == "telegram" {
|
||||
return []string{"novox"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if got, want := planLineOn(p, now, tierAtLeast, running),
|
||||
"mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want {
|
||||
t.Fatalf("the line reads %q, want %q", got, want)
|
||||
}
|
||||
if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") {
|
||||
t.Fatalf("without the machines the line reads %q", got)
|
||||
}
|
||||
old := p
|
||||
old.Commits, old.Delivery = nil, nil
|
||||
if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") {
|
||||
t.Fatalf("a record naming no pull request reads %q", got)
|
||||
}
|
||||
long := p
|
||||
long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6)
|
||||
if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") {
|
||||
t.Fatalf("a long title is not cut at fifty runes: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it.
|
||||
func sameMerge(a, b inventory.PlanMerge) bool {
|
||||
return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried
|
||||
}
|
||||
|
||||
// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched
|
||||
// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder:
|
||||
// its delivery would skip its turn. It joins the next catalogue batch.
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still
|
||||
// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18),
|
||||
// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19).
|
||||
|
||||
// The kinds S18 and S19 raise.
|
||||
const (
|
||||
kindBatchNotCut = "batch-not-cut"
|
||||
kindBatchBehindWalk = "batch-behind-walk"
|
||||
// kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after
|
||||
// its first (novox/hq ADR 0418 clause 3).
|
||||
kindGroupNotMerged = "group-not-merged"
|
||||
)
|
||||
|
||||
// batchFacts is one batch not yet cut, as the watchdogs read it.
|
||||
type batchFacts struct {
|
||||
id, state, grouped string
|
||||
// atMost is when its window closes at the latest; closed when it closed.
|
||||
atMost, closed time.Time
|
||||
// behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its
|
||||
// tiers.
|
||||
behind string
|
||||
walkBound time.Duration
|
||||
// holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first
|
||||
// merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting).
|
||||
holds map[string]string
|
||||
created time.Time
|
||||
atMostWindow time.Duration
|
||||
}
|
||||
|
||||
// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind.
|
||||
func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) {
|
||||
batches, err := inv.Batches(ctx)
|
||||
if err != nil || len(batches) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
bounds, err := measuredTierBounds(ctx, inv, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []batchFacts
|
||||
for _, b := range batches {
|
||||
_, atMostWindow := mergeWindowOf()
|
||||
f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow}
|
||||
if w := b.Delivery; w != nil && w.Batch != nil {
|
||||
f.holds = w.Batch.Holds
|
||||
f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind
|
||||
if f.atMost.Before(f.closed) {
|
||||
f.closed = f.atMost
|
||||
}
|
||||
if w.Batch.CutAtOnce && !b.Created.IsZero() {
|
||||
// Its window closed when it was made, which is its latest too (ADR 0304 decision 1).
|
||||
f.atMost, f.closed = b.Created, b.Created
|
||||
}
|
||||
}
|
||||
if f.behind != "" {
|
||||
if walk, err := inv.PlanByID(ctx, f.behind); err == nil {
|
||||
f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers)))
|
||||
}
|
||||
}
|
||||
out = append(out, f)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open.
|
||||
func watchBatchesNotCut(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter ||
|
||||
len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition
|
||||
continue
|
||||
}
|
||||
late := f.now.Sub(b.atMost)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+
|
||||
"the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped),
|
||||
Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)),
|
||||
Headline: "Merged changes are not being delivered",
|
||||
Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+
|
||||
"Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)),
|
||||
Resolved: "Merged changes are being delivered again"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it.
|
||||
func watchBatchesBehind(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound {
|
||||
continue
|
||||
}
|
||||
in := f.now.Sub(b.closed)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+
|
||||
"grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound),
|
||||
b.grouped, b.behind),
|
||||
Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)),
|
||||
Headline: "Merged changes wait behind a slow delivery",
|
||||
Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+
|
||||
"them, which is taking longer than it should. Nothing is lost.", humanDuration(in)),
|
||||
Resolved: "Merged changes no longer wait behind a slow delivery"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge
|
||||
// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of
|
||||
// the group is delivered while it waits.
|
||||
func watchGroupsNotMerged(f *signalFacts) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, b := range f.batches {
|
||||
if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow {
|
||||
continue
|
||||
}
|
||||
heads := make([]string, 0, len(b.holds))
|
||||
for h := range b.holds {
|
||||
heads = append(heads, h)
|
||||
}
|
||||
sort.Strings(heads)
|
||||
var why []string
|
||||
for _, h := range heads {
|
||||
why = append(why, b.holds[h])
|
||||
}
|
||||
waited := f.now.Sub(b.created)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id,
|
||||
Kind: kindGroupNotMerged, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+
|
||||
"of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id),
|
||||
Said: "a group waits for a member that has not merged",
|
||||
Headline: "A change waits for the rest of its group to merge",
|
||||
Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+
|
||||
"One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.",
|
||||
humanDuration(waited)),
|
||||
Resolved: "The group's last change merged"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once
|
||||
// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery.
|
||||
func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
// Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in
|
||||
// builder's key done at once, one step would carry a one-module walk to done.
|
||||
wholeKeys, held := openPlanKeys, &atomic.Bool{}
|
||||
held.Store(true)
|
||||
openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
all, done, err := wholeKeys()
|
||||
return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err
|
||||
}
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() {
|
||||
t.Fatalf("the first walk is not open: %+v", ws)
|
||||
}
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("two", "c2", heard)), heard)
|
||||
advanceHeld(ctx, open)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 0 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs))
|
||||
}
|
||||
for _, w := range ws {
|
||||
if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) {
|
||||
t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(),
|
||||
deferred(w), askedAny(w), w.Note)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is
|
||||
// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973).
|
||||
func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }()
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("one", "c2", heard)), heard)
|
||||
hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second))
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 {
|
||||
t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs))
|
||||
}
|
||||
b := bs[0]
|
||||
words := batchWords(b, heard)
|
||||
if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" ||
|
||||
!strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") {
|
||||
t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words)
|
||||
}
|
||||
if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() {
|
||||
t.Fatalf("the started walk was %s by the related merge", got.State)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(2*time.Second))
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs)
|
||||
}
|
||||
}
|
||||
|
||||
// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever
|
||||
// the two share (issue 973): it replaces the controller, so it goes alone.
|
||||
func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) {
|
||||
open := windowed(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
hear(t, open, repoMerge("one", "c1", t0), t0)
|
||||
cutAt(t, open, t0.Add(90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
first := ws[0]
|
||||
heard := t0.Add(100 * time.Second)
|
||||
hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard)
|
||||
ws, bs := walks(t, open)
|
||||
if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued ||
|
||||
bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") {
|
||||
t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
first.State = inventory.PlanDone
|
||||
if err := open.inventory.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutAt(t, open, heard.Add(time.Second))
|
||||
ws, _ = walks(t, open)
|
||||
if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" {
|
||||
t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws)
|
||||
}
|
||||
own := ws[0]
|
||||
// Unrelated to the own-path walk by every footprint, a merge still waits for it.
|
||||
later := heard.Add(time.Minute)
|
||||
hear(t, open, lone(repoMerge("two", "c2", later)), later)
|
||||
ws, bs = walks(t, open)
|
||||
if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID {
|
||||
t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs)
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,21 @@ func otherMachines(machine string, nodes ...string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
|
||||
// condition at once where this process keeps the conditions: a push is when a person is looking.
|
||||
func reportKept(ctx context.Context, plan catalogue.Resolution) {
|
||||
for _, k := range plan.Kept {
|
||||
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
|
||||
if conditionsFrom != nil {
|
||||
o := keptObservation(k)
|
||||
o.Source = probeBindingsID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
|
||||
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
|
||||
// moves it (said, so the data goes first), and never anything else.
|
||||
|
||||
@@ -75,3 +75,19 @@ func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
|
||||
t.Error("two consumers, one condition")
|
||||
}
|
||||
}
|
||||
|
||||
// A push says the move it refused, and raises its condition at once.
|
||||
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
|
||||
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
|
||||
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
|
||||
open[0].Source != probeBindingsID {
|
||||
t.Fatalf("raised %+v", open)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +150,16 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild)
|
||||
return notRegistered(err)
|
||||
}
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
||||
return manifest, err
|
||||
} else if failed {
|
||||
return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", j.by, manifest.Module,
|
||||
short(j.commit))
|
||||
}
|
||||
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
||||
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if j.carried != nil {
|
||||
@@ -219,6 +229,10 @@ type answers struct {
|
||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||
// recorded at send, not at apply).
|
||||
reported []inventory.Reported
|
||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
||||
plans []inventory.Plan
|
||||
// tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296).
|
||||
tierBounds tierBounds
|
||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||
// other answer here: those are about a machine that was told something, and this is about one
|
||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||
@@ -273,4 +287,8 @@ type answers struct {
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
||||
// issue 485); lastSentUnread why it could not be read.
|
||||
lastSent map[string]inventory.Send
|
||||
lastSentUnread string
|
||||
}
|
||||
|
||||
@@ -524,6 +524,7 @@ type registrations interface {
|
||||
SourceIdentity(ctx context.Context, module string) (int64, error)
|
||||
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
||||
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
||||
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
||||
}
|
||||
|
||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||
|
||||
@@ -129,6 +129,41 @@ func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
@@ -270,11 +305,10 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
var forgeMerge keptMerge
|
||||
_ = json.Unmarshal(event, &forgeMerge)
|
||||
wasForge := keptForgeMerge
|
||||
t.Cleanup(func() { keptForgeMerge = wasForge })
|
||||
keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil }
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
||||
|
||||
@@ -47,11 +47,6 @@ func aBuilderOnTheBus(t *testing.T) builderOnTheBus {
|
||||
}
|
||||
*c.into = kv
|
||||
}
|
||||
// And the controller's own buckets, as its start asserts them: a merge it hears is kept in the bucket of the merges
|
||||
// (ADR 0539 §17), which no other test need have made on this bus first.
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
withTrunkList(t, true)
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1945 (dev-r6-controller's B8 follows it): what a bus machine runs is read from its have, not from a send kept, so a
|
||||
// machine whose have names the build the mesh holds is not moved by a bus upgrade, and one whose have names an older
|
||||
// build is.
|
||||
func TestABusMachineRunsWhatItsHaveNames(t *testing.T) {
|
||||
was := busRuns
|
||||
t.Cleanup(func() { busRuns = was })
|
||||
busRuns = func(_ context.Context, node, _ string) (string, bool, error) {
|
||||
return map[string]string{"novox": "src2:new", "ace": "src2:old"}[node], node != "g14", nil
|
||||
}
|
||||
b := busPending{module: "nats", to: "c-new", from: map[string]string{}, same: map[string]bool{}}
|
||||
commitOf := map[string]string{"src2:new": "c-new", "src2:old": "c-old"}
|
||||
for _, n := range []string{"novox", "ace", "g14"} {
|
||||
fp, known, _ := busRuns(context.Background(), n, "nats")
|
||||
if c, ok := commitOf[fp]; known && ok {
|
||||
b.from[n] = c
|
||||
}
|
||||
}
|
||||
if b.moves("novox") || !b.moves("ace") || !b.moves("g14") {
|
||||
t.Errorf("novox moves %v (runs the new build), ace %v (an older one), g14 %v (says none)", b.moves("novox"),
|
||||
b.moves("ace"), b.moves("g14"))
|
||||
}
|
||||
}
|
||||
+12
-106
@@ -10,10 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -113,44 +111,20 @@ func pendingBus(ctx context.Context, inv *inventory.Inventory) (busPending, erro
|
||||
b.same[commit] = true
|
||||
}
|
||||
}
|
||||
// What each machine runs is its have's fingerprint (novox/hq ADR 0539 clause 9), the commit of the build of it.
|
||||
commitOf := map[string]string{}
|
||||
for commit, src := range sources {
|
||||
if src != "" {
|
||||
commitOf[src] = commit
|
||||
}
|
||||
}
|
||||
for _, n := range b.machines {
|
||||
fp, known, err := busRuns(ctx, n, b.module)
|
||||
sent, known, err := inv.SentBuilds(ctx, n)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
if c, ok := commitOf[fp]; known && ok {
|
||||
b.from[n] = c
|
||||
if known {
|
||||
if c, carried := sent[b.module]; carried {
|
||||
b.from[n] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// busRuns is the fingerprint a machine's node-engine says it runs of a module, from its have; known is false when it
|
||||
// says none. A variable so a test reads it without a bus.
|
||||
var busRuns = func(ctx context.Context, node, module string) (string, bool, error) {
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
h, found, err := haveOn(ctx, js, node, module)
|
||||
if err != nil || !found || h.FP == "" {
|
||||
return "", false, err
|
||||
}
|
||||
return h.FP, true, nil
|
||||
}
|
||||
|
||||
// busHeld names the machines a push may not send because sending them would replace the bus: the
|
||||
// planned step's, not a push's (ADR 0236). Said with the remedy.
|
||||
func busHeld(ctx context.Context, inv *inventory.Inventory, machines []string) (map[string]string, error) {
|
||||
@@ -255,85 +229,17 @@ func busCommand(ctx context.Context, args []string) error {
|
||||
fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From,
|
||||
short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it",
|
||||
false: "NOT reversible: the snapshot is the only way back"}[*reversible])
|
||||
// **Nothing is sent, and the assigner never moves the bus** (novox/hq ADR 0539 clause 15): the bus's machines are
|
||||
// composed at the build the mesh holds, and the operator writes each machine's want with the operator's credential,
|
||||
// recorded, naming that composition.
|
||||
wants, err := busWantsToWrite(withBusStep(ctx), open, b.module, moving)
|
||||
sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving)
|
||||
if err != nil {
|
||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the bus's composition could not be written: "+err.Error())
|
||||
return fmt.Errorf("the bus's machines could not be composed at its new build: %w — nothing was replaced", err)
|
||||
}
|
||||
fmt.Printf("composed %s at the bus's new build. Write each want below with the operator's credential, and record it "+
|
||||
"in the hand-act log; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot as the "+
|
||||
"way back. `bus` says how it went\n", strings.Join(moving, ", "), busStepBound)
|
||||
for _, w := range wants {
|
||||
fmt.Println(" " + w)
|
||||
_ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error())
|
||||
return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err)
|
||||
}
|
||||
fmt.Printf("sent %s; `bus-maintenance` is open until the bus answers healthy again — every stream, every durable "+
|
||||
"consumer, a round trip to the machines (H-bus) — within %s, or the step is said failed with its snapshot "+
|
||||
"as the way back. `bus` says how it went\n", strings.Join(sent, ", "), busStepBound)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busWantsToWrite composes the bus's machines and answers, for each, the want the operator writes: its key in the
|
||||
// assigner's bucket and its value, naming the fingerprint the bus module is registered at and that machine's
|
||||
// composition of it.
|
||||
func busWantsToWrite(ctx context.Context, open *stores, module string, machines []string) ([]string, error) {
|
||||
if _, err := composeModules(ctx, open, machines); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fps, err := currentFingerprints(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
digestOf := func(node string) (string, error) {
|
||||
key, err := sdkdelivery.ComposedKey(node, module, fps[module])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
e, err := digests.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
v, err := sdkdelivery.DecodeComposedDigest(e.Value())
|
||||
return v.Digest, err
|
||||
}
|
||||
return busWants(module, fps[module], machines, digestOf)
|
||||
}
|
||||
|
||||
// busWants is each machine's want for the bus module, as the operator writes it: `<bucket> <key> <value>`.
|
||||
func busWants(module, fp string, machines []string, digestOf func(node string) (string, error)) ([]string, error) {
|
||||
var out []string
|
||||
for _, node := range machines {
|
||||
digest, err := digestOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := sdkdelivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := sdkdelivery.EncodeWant(sdkdelivery.Want{FP: fp, Composed: digest,
|
||||
Words: "the bus moved by a person's step (bus upgrade, ADR 0539 clause 15)"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s %s %s", sdkdelivery.AssignerWantsBucket, key, value))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busStatus is `bus`: what an upgrade would do, and the last step.
|
||||
func busStatus(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
@@ -391,7 +297,7 @@ func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reports, err := lastReports(ctx, inv)
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1945/AC7: a bus upgrade answers, for each machine, the want the operator writes: the assigner's bucket, the
|
||||
// machine's key for the bus module, and a want naming the bus's fingerprint and that machine's composition of it. It
|
||||
// sends nothing: nothing it calls publishes on a node's declaration subject (busWants composes no send at all).
|
||||
func TestABusUpgradeAnswersTheWantTheOperatorWrites(t *testing.T) {
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
digests := map[string]string{"novox": "sha256:" + strings.Repeat("1", 64), "ace": "sha256:" + strings.Repeat("2", 64)}
|
||||
lines, err := busWants("nats", fp, []string{"ace", "novox"}, func(node string) (string, error) { return digests[node], nil })
|
||||
if err != nil || len(lines) != 2 {
|
||||
t.Fatalf("busWants = %v, %v", lines, err)
|
||||
}
|
||||
for i, node := range []string{"ace", "novox"} {
|
||||
parts := strings.SplitN(lines[i], " ", 3)
|
||||
if len(parts) != 3 || parts[0] != sdkdelivery.AssignerWantsBucket || parts[1] != "want."+node+".nats" {
|
||||
t.Fatalf("line %q", lines[i])
|
||||
}
|
||||
w, err := sdkdelivery.DecodeWant([]byte(parts[2]))
|
||||
if err != nil || w.FP != fp || w.Composed != digests[node] {
|
||||
t.Errorf("%s's want = %+v, %v", node, w, err)
|
||||
}
|
||||
}
|
||||
if _, err := busWants("nats", fp, []string{"ace"}, func(string) (string, error) { return "", nil }); err == nil {
|
||||
t.Errorf("a want naming no composition was answered")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A send held for the bus's planned step (novox/hq issue 336).
|
||||
//
|
||||
// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned
|
||||
// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every
|
||||
// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only
|
||||
// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move.
|
||||
// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what
|
||||
// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it
|
||||
// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the
|
||||
// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks'
|
||||
// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB).
|
||||
|
||||
// kindBusStepWaiting is S17's kind: bus.<module>.step-waiting.
|
||||
const kindBusStepWaiting = "bus-step-waiting"
|
||||
|
||||
// busFacts is a new bus build waiting for its planned step, and the sends held for it.
|
||||
type busFacts struct {
|
||||
module, to string
|
||||
// from is the build each machine of the bus runs; machines those whose bus the step would replace.
|
||||
from map[string]string
|
||||
machines []string
|
||||
waits []busWaitFacts
|
||||
}
|
||||
|
||||
// busWaitFacts is one walk whose send was refused because it would replace the bus.
|
||||
type busWaitFacts struct {
|
||||
plan, repository, commit string
|
||||
// modules are what its tier sends: what waits.
|
||||
modules []string
|
||||
// since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal.
|
||||
since time.Time
|
||||
}
|
||||
|
||||
// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note
|
||||
// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save.
|
||||
var busRefusedFirst = &firstSeen{at: map[string]time.Time{}}
|
||||
|
||||
type firstSeen struct {
|
||||
mu sync.Mutex
|
||||
at map[string]time.Time
|
||||
}
|
||||
|
||||
// mark keeps the first moment an id was seen.
|
||||
func (s *firstSeen) mark(id string, at time.Time) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, seen := s.at[id]; !seen {
|
||||
s.at[id] = at
|
||||
}
|
||||
}
|
||||
|
||||
// of is when an id was first seen; zero when it was not.
|
||||
func (s *firstSeen) of(id string) time.Time {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.at[id]
|
||||
}
|
||||
|
||||
// keepOnly forgets every id not given: a walk no longer held is not held since then.
|
||||
func (s *firstSeen) keepOnly(ids map[string]bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id := range s.at {
|
||||
if !ids[id] {
|
||||
delete(s.at, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step.
|
||||
func refusedForTheBus(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), errBusWaits.Error())
|
||||
}
|
||||
|
||||
// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its
|
||||
// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace
|
||||
// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not.
|
||||
func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts {
|
||||
f := busFacts{module: b.module, to: b.to, from: b.from}
|
||||
for _, n := range b.machines {
|
||||
if b.moves(n) {
|
||||
f.machines = append(f.machines, n)
|
||||
}
|
||||
}
|
||||
if len(f.machines) == 0 {
|
||||
return f
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) {
|
||||
continue
|
||||
}
|
||||
since := p.Updated
|
||||
if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) {
|
||||
since = seen
|
||||
}
|
||||
var modules []string
|
||||
if p.Tier >= 0 && p.Tier < len(p.Tiers) {
|
||||
modules = append(modules, p.Tiers[p.Tier]...)
|
||||
} else {
|
||||
modules = planModules(p)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules,
|
||||
since: since})
|
||||
}
|
||||
sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) })
|
||||
return f
|
||||
}
|
||||
|
||||
// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out.
|
||||
func (b busFacts) heldByTheBus() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, w := range b.waits {
|
||||
out[w.plan] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with
|
||||
// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true").
|
||||
const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`"
|
||||
|
||||
// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator.
|
||||
func watchBusWaits(f *signalFacts) []conditions.Observation {
|
||||
b := f.bus
|
||||
if len(b.waits) == 0 || len(b.machines) == 0 {
|
||||
return nil
|
||||
}
|
||||
since := b.waits[0].since
|
||||
var walks, what []string
|
||||
for _, w := range b.waits {
|
||||
walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit),
|
||||
strings.Join(w.modules, ", ")))
|
||||
what = append(what, deliveryWhat(w.modules, w.repository))
|
||||
}
|
||||
var from []string
|
||||
for _, n := range b.machines {
|
||||
from = append(from, short(orNotKnown(b.from[n])))
|
||||
}
|
||||
machines := namesWords(b.machines, 3)
|
||||
in := f.now.Sub(since)
|
||||
return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting",
|
||||
Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
||||
Machine: b.machines[0], Also: b.machines[1:],
|
||||
Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+
|
||||
"the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "),
|
||||
b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb,
|
||||
since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")),
|
||||
Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)),
|
||||
Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax),
|
||||
Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+
|
||||
"installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines,
|
||||
namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax),
|
||||
Needs: "start the bus upgrade " + FromMeshMCPServer,
|
||||
Resolved: "Resolved: the bus upgrade started, and sends go on"}}
|
||||
}
|
||||
|
||||
// sortedUnique is a list sorted, each once.
|
||||
func sortedUnique(xs []string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, x := range xs {
|
||||
if !seen[x] {
|
||||
seen[x] = true
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// clipWords keeps a plain sentence within a bound, at a word.
|
||||
func clipWords(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := strings.LastIndex(s[:n-1], " ")
|
||||
if cut <= 0 {
|
||||
cut = n - 1
|
||||
}
|
||||
return s[:cut] + "…"
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
walk := func(id, repo, note string, updated time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0,
|
||||
Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note,
|
||||
Updated: updated}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
bus busPending
|
||||
plans []inventory.Plan
|
||||
first map[string]time.Time
|
||||
want []string
|
||||
since time.Time
|
||||
}{
|
||||
{"a walk refused for the bus", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil,
|
||||
[]string{"plan-1"}, now.Add(-27 * time.Minute)},
|
||||
{"refused earlier than its last save, as this controller saw it", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))},
|
||||
map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)},
|
||||
{"a walk refused for another reason", b,
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil,
|
||||
nil, time.Time{}},
|
||||
{"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(),
|
||||
[]inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"the bus already runs the build held: its step started", func() busPending {
|
||||
o := aBusOnAnchor()
|
||||
o.from = map[string]string{"anchor": o.to}
|
||||
return o
|
||||
}(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}},
|
||||
{"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan {
|
||||
p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)
|
||||
p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"}
|
||||
return []inventory.Plan{p}
|
||||
}(), nil, nil, time.Time{}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] })
|
||||
var ids []string
|
||||
for _, w := range got.waits {
|
||||
ids = append(ids, w.plan)
|
||||
}
|
||||
if strings.Join(ids, ",") != strings.Join(c.want, ",") {
|
||||
t.Fatalf("held for the bus: %v, want %v", ids, c.want)
|
||||
}
|
||||
if len(c.want) > 0 {
|
||||
if !got.waits[0].since.Equal(c.since) {
|
||||
t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since)
|
||||
}
|
||||
if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" {
|
||||
t.Errorf("behind %+v", got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a
|
||||
// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that
|
||||
// ends it — and the wait is not read as a walk running late.
|
||||
|
||||
// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held.
|
||||
func aBusOnAnchor() busPending {
|
||||
return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"},
|
||||
to: "32307bd1bbbb", same: map[string]bool{}}
|
||||
}
|
||||
|
||||
// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it.
|
||||
func refusedNote(b busPending, tier int) string {
|
||||
held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) +
|
||||
"), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)"
|
||||
return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again"
|
||||
}
|
||||
|
||||
// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises
|
||||
// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to,
|
||||
// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the
|
||||
// condition clears once the bus's machine runs the new build.
|
||||
func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) {
|
||||
now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC)
|
||||
b := aBusOnAnchor()
|
||||
held := func(entered time.Duration) *signalFacts {
|
||||
f := calm(now)
|
||||
f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1,
|
||||
entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}}
|
||||
f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"},
|
||||
waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122",
|
||||
modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}}
|
||||
return f
|
||||
}
|
||||
|
||||
f := held(time.Minute)
|
||||
got := watchBusWaits(f)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a send held for the bus a minute ago raised %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver)
|
||||
}
|
||||
for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus",
|
||||
`"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Errorf("its summary does not say %q: %s", want, o.Summary)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" {
|
||||
t.Errorf("its words do not say what waits and what the operator does: %+v", o)
|
||||
}
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation,
|
||||
Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok {
|
||||
t.Errorf("its words are not plain: %s", why)
|
||||
}
|
||||
|
||||
// Past S3's bound: still the bus's wait, never a stalled walk.
|
||||
late := held(45 * time.Minute)
|
||||
if s3 := watchPlans(late); len(s3) != 0 {
|
||||
t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3)
|
||||
}
|
||||
// A walk held for something else past its bound is still stalled.
|
||||
other := held(45 * time.Minute)
|
||||
other.plans[0].bus = false
|
||||
if s3 := watchPlans(other); len(s3) != 1 {
|
||||
t.Fatalf("a walk held for something else past its bound raised %+v", s3)
|
||||
}
|
||||
|
||||
// Through the keeper: open at the first tick, cleared when the step started.
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{},
|
||||
Now: func() time.Time { return now }})
|
||||
defer k.Close(context.Background())
|
||||
w := &watchdogs{keeper: k, started: now.Add(-time.Hour)}
|
||||
w.see(t.Context(), held(time.Minute))
|
||||
open, err := k.Open(t.Context())
|
||||
if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" {
|
||||
t.Fatalf("after the first tick, open: %+v (%v)", open, err)
|
||||
}
|
||||
started := held(time.Minute)
|
||||
started.bus = busFacts{module: "nats", to: b.to}
|
||||
started.plans[0].bus = false
|
||||
w.see(t.Context(), started)
|
||||
if open, _ := k.Open(t.Context()); len(open) != 0 {
|
||||
t.Fatalf("the step started, and open: %+v", open)
|
||||
}
|
||||
}
|
||||
@@ -29,13 +29,7 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
// Once every node runs the node-engine that reads no declaration, the declarations' stream and the nodes'
|
||||
// consumers on it are not asserted again (novox/hq ADR 0539 clause 11; composed_start.go retires them).
|
||||
raise := func() error { return broker.Raise(r, names) }
|
||||
if declarationsRetiredNow(ctx, inv) {
|
||||
raise = func() error { return broker.RaiseAfterDeclarations(r) }
|
||||
}
|
||||
if err := raise(); err != nil {
|
||||
if err := broker.Raise(r, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
|
||||
|
||||
@@ -7,9 +7,14 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
@@ -160,3 +165,43 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
// The controller starting, before either was assigned.
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, nil)
|
||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||
t.Fatal("a send that asserted everything raised a condition")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
|
||||
//
|
||||
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
|
||||
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
|
||||
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
|
||||
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
|
||||
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
|
||||
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
|
||||
// its gate starts as before.
|
||||
|
||||
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
|
||||
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
|
||||
var busyGateSlack = 5 * time.Minute
|
||||
|
||||
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
|
||||
// controller's clock from when it first read the wait.
|
||||
const busyEngineBound = 15 * time.Minute
|
||||
|
||||
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
|
||||
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
|
||||
// machine nor a wrong word from it holds a walk longer.
|
||||
func busyGateEnds(b *inventory.GateBusy) time.Time {
|
||||
until := b.Seen.Add(busyEngineBound)
|
||||
if !b.Until.IsZero() && b.Until.Before(until) {
|
||||
until = b.Until
|
||||
}
|
||||
return until.Add(busyGateSlack)
|
||||
}
|
||||
|
||||
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
|
||||
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
|
||||
// gate's Since to its end. Pure but for the gate it is handed.
|
||||
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
|
||||
if g.Busy != nil && g.Busy.Ended != nil {
|
||||
return false
|
||||
}
|
||||
for _, j := range pairs {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
continue
|
||||
}
|
||||
if g.Busy == nil {
|
||||
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
|
||||
Seen: now}
|
||||
}
|
||||
if g.Busy.Seen.IsZero() {
|
||||
g.Busy.Seen = now
|
||||
}
|
||||
g.Busy.Until, g.Busy.Said = b.Until, b.Said
|
||||
if now.After(busyGateEnds(g.Busy)) {
|
||||
// The engine said it would have replaced it by now: the judging starts, and finds the module not
|
||||
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
|
||||
end := now
|
||||
g.Busy.Ended, g.Busy.PastBound = &end, true
|
||||
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
|
||||
return false
|
||||
}
|
||||
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
|
||||
g.Read(now, false, g.Busy.Words())
|
||||
return true
|
||||
}
|
||||
if g.Busy == nil {
|
||||
return false
|
||||
}
|
||||
// Replaced: the judging starts at the report that said so.
|
||||
end := now
|
||||
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
|
||||
end = r.At.UTC()
|
||||
}
|
||||
g.Busy.Ended = &end
|
||||
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
|
||||
return false
|
||||
}
|
||||
|
||||
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
|
||||
// running because it said busy.
|
||||
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
|
||||
r, said := f.reports[j.node]
|
||||
if !said || r.At == nil || !f.reportedOn(j.node, r) {
|
||||
return inventory.BusyWait{}, false
|
||||
}
|
||||
return inventory.WaitingFor(r.Busy, j.module)
|
||||
}
|
||||
|
||||
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
|
||||
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
|
||||
func busyStill(j judged, f gateFacts) (string, bool) {
|
||||
b, waits := busyOn(j, f)
|
||||
if !waits {
|
||||
return "", false
|
||||
}
|
||||
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
|
||||
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
|
||||
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
|
||||
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
|
||||
reported := sent.Add(20 * time.Second)
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
|
||||
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
|
||||
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
|
||||
// As judgeMoves hands it: the report held against the gate's own send.
|
||||
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
|
||||
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
|
||||
}
|
||||
|
||||
now := reported.Add(time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported, waiting), now) {
|
||||
t.Fatal("a gate whose module's container waits as busy judged it")
|
||||
}
|
||||
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
|
||||
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
|
||||
}
|
||||
for _, said := range []string{gateLine(g), g.Last} {
|
||||
if !strings.Contains(said, "waits for a busy container on novox") {
|
||||
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(g.Last, "check of #280") {
|
||||
t.Errorf("the wait does not say what the container holds: %s", g.Last)
|
||||
}
|
||||
|
||||
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
|
||||
now = reported.Add(10 * time.Minute)
|
||||
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
|
||||
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
|
||||
}
|
||||
|
||||
// The machine reports it replaced: the judging starts at that report.
|
||||
replacedAt := reported.Add(12 * time.Minute)
|
||||
now = replacedAt.Add(5 * time.Second)
|
||||
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
|
||||
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
|
||||
t.Fatal("the gate still waits after the container was replaced")
|
||||
}
|
||||
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
|
||||
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
|
||||
}
|
||||
if strings.Contains(gateLine(g), "waits for a busy container") {
|
||||
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
|
||||
}
|
||||
// And a later report listing it waiting again does not hold a gate that started.
|
||||
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
|
||||
t.Error("a gate that started waits again")
|
||||
}
|
||||
}
|
||||
|
||||
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
|
||||
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
|
||||
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &sent, Current: true, Busy: waiting}}}
|
||||
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
|
||||
if busyWait(g, pairs, f, now) {
|
||||
t.Fatal("a gate waited past the engine's bound and the slack")
|
||||
}
|
||||
if !g.Busy.PastBound || !g.Since.Equal(now) {
|
||||
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
|
||||
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
|
||||
// report not about the gate's send says nothing of it.
|
||||
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
|
||||
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
|
||||
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
|
||||
pairs := []judged{{module: "build-agent", node: "ace"}}
|
||||
at := sent.Add(time.Minute)
|
||||
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d2", ReportedSequence: 9}}}
|
||||
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
|
||||
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
|
||||
}
|
||||
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
|
||||
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
|
||||
if busyWait(g, pairs, elsewhere, at) {
|
||||
t.Fatal("a wait reported about an earlier declaration held the gate")
|
||||
}
|
||||
}
|
||||
|
||||
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
|
||||
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
|
||||
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
|
||||
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
|
||||
pairs := []judged{{module: "build-agent", node: "novox"}}
|
||||
ahead := seen.Add(time.Hour)
|
||||
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
|
||||
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
|
||||
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
|
||||
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
|
||||
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
|
||||
}
|
||||
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
|
||||
t.Fatal("the gate stopped waiting within the bound on its own clock")
|
||||
}
|
||||
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
|
||||
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
|
||||
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
|
||||
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
|
||||
{"id": "agent-env", "type": "file"}}}}
|
||||
lines := busyNotSent(modules, false)
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
|
||||
!strings.Contains(lines[0], "replaced at once") {
|
||||
t.Fatalf("the plan does not say the busy not sent: %v", lines)
|
||||
}
|
||||
if lines := busyNotSent(modules, true); len(lines) != 0 {
|
||||
t.Fatalf("a busy sent is said as not sent: %v", lines)
|
||||
}
|
||||
}
|
||||
@@ -24,4 +24,28 @@ func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
||||
}
|
||||
if got := answersFirst(argv); got != c.want {
|
||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
|
||||
@@ -0,0 +1,643 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **The controller catches up on merges it did not hear** (novox/hq ADR 0457, issue 1336). A merge the forge announced while the
|
||||
// controller was being replaced was never heard, and nothing asked for it again: the trunk held a merge no walk answered. At its
|
||||
// start, and each time it hears a merge, the controller reads the last catchUpWindow merge commits of a trunk from the forge, and
|
||||
// hears each one it holds no record of as if the forge had announced it then, on the warrant the router holds for it.
|
||||
// It lasts until release 6 of design 47 deletes delivery plans and batches (ADR 0429 §7, §8).
|
||||
|
||||
// catchUpWindow is how many of a trunk's newest merge commits are read; an older merge is a person's to decide.
|
||||
const catchUpWindow = 50
|
||||
|
||||
// kindMergeNotHeard is the condition of a merge the controller did not hear and holds no warrant to plan.
|
||||
const kindMergeNotHeard = "merge-not-heard"
|
||||
|
||||
// trunkMerge is a merge commit of a trunk as the forge lists it.
|
||||
type trunkMerge struct {
|
||||
Commit string
|
||||
// Head is the merge commit's second parent: the pull request's head that was merged.
|
||||
Head string
|
||||
Subject string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// trunkCommit is a commit of a trunk as the forge lists it, newest first: a merge commit has a second parent.
|
||||
type trunkCommit struct {
|
||||
Commit, SecondParent, Subject string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// trunkCommitsPage is one page (fifty) of a trunk's commits, newest first; empty past the last. A variable so a test needs no
|
||||
// forge.
|
||||
var trunkCommitsPage = func(ctx context.Context, owner, repo, branch string, page int) ([]trunkCommit, error) {
|
||||
var found struct {
|
||||
Result []struct {
|
||||
SHA string `json:"sha"`
|
||||
Parents []struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"parents"`
|
||||
Commit struct {
|
||||
Message string `json:"message"`
|
||||
Committer struct {
|
||||
Date string `json:"date"`
|
||||
} `json:"committer"`
|
||||
} `json:"commit"`
|
||||
} `json:"result"`
|
||||
}
|
||||
apiPath := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&limit=50&page=%d&stat=false&verification=false&files=false",
|
||||
owner, repo, url.QueryEscape(branch), page)
|
||||
missing, err := askForgeApi(ctx, apiPath, &found)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if missing {
|
||||
return nil, fmt.Errorf("the forge holds no branch %s of %s/%s", branch, owner, repo)
|
||||
}
|
||||
var out []trunkCommit
|
||||
for _, c := range found.Result {
|
||||
at, _ := time.Parse(time.RFC3339, c.Commit.Committer.Date)
|
||||
tc := trunkCommit{Commit: c.SHA, Subject: firstLine(c.Commit.Message), At: at.UTC()}
|
||||
if len(c.Parents) >= 2 {
|
||||
tc.SecondParent = c.Parents[1].SHA
|
||||
}
|
||||
out = append(out, tc)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// readTrunkMerges is the last catchUpWindow merge commits of a repository's trunk, oldest first: pages are read until that many
|
||||
// merges are held or the trunk ends.
|
||||
func readTrunkMerges(ctx context.Context, owner, repo, branch string) ([]trunkMerge, error) {
|
||||
var merges []trunkMerge
|
||||
for page := 1; page <= 8 && len(merges) < catchUpWindow; page++ {
|
||||
commits, err := trunkCommitsPage(ctx, owner, repo, branch, page)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(commits) == 0 {
|
||||
break
|
||||
}
|
||||
for _, c := range commits {
|
||||
// A merge of the trunk is a pull request's merge into it. A commit with a second parent that is none is a merge made
|
||||
// inside some branch ("Merge remote-tracking branch 'origin/main' into fix/..."), which the trunk's history lists too and
|
||||
// which no Approve ever covered: read as a gap it would raise a condition for nothing a person decides.
|
||||
if c.SecondParent != "" && mergesIntoTrunk(c.Subject, branch) {
|
||||
merges = append(merges, trunkMerge{Commit: c.Commit, Head: c.SecondParent, Subject: c.Subject, At: c.At})
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastMerges(merges), nil
|
||||
}
|
||||
|
||||
// sinceTheRecord are the merges that can be gaps (novox/hq ADR 0463 §2): those made since the controller's record of merges begins on
|
||||
// the mesh (first, the earliest merge it holds of any repository and branch, less mergeGrace as owedLate judges). A merge before it is
|
||||
// history on every trunk, and a trunk with no merge kept of its own still has the merges made since as gaps. With no merge kept
|
||||
// anywhere there is no record to have a gap in.
|
||||
func sinceTheRecord(merges []trunkMerge, kept bool, first time.Time) []trunkMerge {
|
||||
if !kept {
|
||||
return nil
|
||||
}
|
||||
var out []trunkMerge
|
||||
for _, m := range merges {
|
||||
if !m.At.Before(first.Add(-mergeGrace)) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// lastMerges is the newest catchUpWindow of merges, given newest first, oldest first.
|
||||
func lastMerges(newestFirst []trunkMerge) []trunkMerge {
|
||||
if len(newestFirst) > catchUpWindow {
|
||||
newestFirst = newestFirst[:catchUpWindow]
|
||||
}
|
||||
out := slices.Clone(newestFirst)
|
||||
slices.Reverse(out)
|
||||
return out
|
||||
}
|
||||
|
||||
var mergeSubject = regexp.MustCompile(`^Merge pull request '(.*)' \(#(\d+)\) from (\S+) into (\S+)$`)
|
||||
|
||||
// pullOfSubject is the pull request a merge commit's subject names: its title and number.
|
||||
func pullOfSubject(subject string) (title string, number int, ok bool) {
|
||||
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
|
||||
if m == nil {
|
||||
return "", 0, false
|
||||
}
|
||||
n, err := strconv.Atoi(m[2])
|
||||
if err != nil || n <= 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
return m[1], n, true
|
||||
}
|
||||
|
||||
// mergesIntoTrunk says a commit's subject is a pull request's merge into this trunk.
|
||||
func mergesIntoTrunk(subject, branch string) bool {
|
||||
m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject))
|
||||
return m != nil && m[4] == branch
|
||||
}
|
||||
|
||||
// headBranchOfSubject is the branch a merge commit's subject says was merged.
|
||||
func headBranchOfSubject(subject string) string {
|
||||
if m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)); m != nil {
|
||||
return m[3]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// readMergeEvent is the merge as the forge would have announced it: the commit, the pull request, the files it changed and the
|
||||
// directories among them that hold a module.json. A variable so a test needs no forge.
|
||||
var readMergeEvent = func(ctx context.Context, owner, repo, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
||||
m := link.SourceMoved{Owner: owner, Repo: repo, Base: branch, Head: headBranchOfSubject(tm.Subject), HeadSHA: tm.Head,
|
||||
Commit: tm.Commit, MergedAt: tm.At.UTC().Format(time.RFC3339)}
|
||||
if title, number, ok := pullOfSubject(tm.Subject); ok {
|
||||
m.Number, m.Title = number, title
|
||||
}
|
||||
var found struct {
|
||||
Result struct {
|
||||
Files []struct {
|
||||
Filename string `json:"filename"`
|
||||
Status string `json:"status"`
|
||||
} `json:"files"`
|
||||
} `json:"result"`
|
||||
}
|
||||
missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(tm.Commit), &found)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
if missing || len(found.Result.Files) == 0 {
|
||||
return m, fmt.Errorf("the forge named no files for the merge %.8s of %s/%s", tm.Commit, owner, repo)
|
||||
}
|
||||
for _, f := range found.Result.Files {
|
||||
m.Paths = append(m.Paths, f.Filename)
|
||||
if f.Status == "removed" || f.Status == "deleted" {
|
||||
m.Removed = append(m.Removed, f.Filename)
|
||||
}
|
||||
}
|
||||
// The forge lists a merge's files by the page; a list as long as a page may be a beginning.
|
||||
m.PathsTruncated = len(m.Paths) >= 300
|
||||
dirs := map[string]bool{}
|
||||
for _, p := range m.Paths {
|
||||
for d := path.Dir(p); ; d = path.Dir(d) {
|
||||
dirs[d] = true
|
||||
if d == "." || d == "/" {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(dirs) > 64 {
|
||||
return m, nil // too many to look at: the old rule stands, a directory is a module when its manifest changed
|
||||
}
|
||||
var holding []string
|
||||
for d := range dirs {
|
||||
file := "module.json"
|
||||
if d != "." {
|
||||
file = d + "/module.json"
|
||||
}
|
||||
_, exists, err := manifestCached(ctx, owner, repo, file, tm.Commit)
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
if exists {
|
||||
holding = append(holding, d)
|
||||
}
|
||||
}
|
||||
sort.Strings(holding)
|
||||
m.ModuleDirs, m.ModuleDirsSaid = holding, true
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// mergeAskPrefix is what the id of a merge ask made by the retired module mesh-delivery begins with: the pull request's number
|
||||
// and the head's first twelve digits (`merge-<number>-<head>-<try>`). The teller's ids name neither (see findMergeWarrant).
|
||||
func mergeAskPrefix(number int, head string) string {
|
||||
if len(head) > 12 {
|
||||
head = head[:12]
|
||||
}
|
||||
return fmt.Sprintf("merge-%d-%s-", number, head)
|
||||
}
|
||||
|
||||
// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking
|
||||
// mesh-delivery's (ADR 0450 §8; issue 1318).
|
||||
func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} }
|
||||
|
||||
// caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant
|
||||
// answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437
|
||||
// §6, as movedOnApprove judges an announced one). The heads the Approve was bound to are checked by the caller, where it can
|
||||
// name them.
|
||||
func caughtUpWarrant(asker string, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
||||
if err := askerAccepted(asker); err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch {
|
||||
case ask == nil || w == nil:
|
||||
return "", errors.New("the router holds no closed record of the ask")
|
||||
case ask.Who != asks.Operator:
|
||||
return "", fmt.Errorf("the ask %s is for %q to answer, not the operator", ask.ID, ask.Who)
|
||||
}
|
||||
option, err := w.For(asker, *ask)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if option.ID != optionApprove || w.Option != optionApprove {
|
||||
return "", fmt.Errorf("the operator chose %q on the ask %s, not an Approve", w.Option, w.Ask)
|
||||
}
|
||||
if w.By == nil || w.By.Who != asks.Operator || w.By.Verified == "" {
|
||||
return "", fmt.Errorf("the ask %s was not answered by the operator on a channel that proves who answered", w.Ask)
|
||||
}
|
||||
return w.Says() + " (ask " + w.Ask + ")", nil
|
||||
}
|
||||
|
||||
// routerAskIDs are the ids of the asks an asker has in the router's record. A variable so a test needs no bus.
|
||||
var routerAskIDs = func(ctx context.Context, inv catalogueReader, asker string) ([]string, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, errors.New("only the serving controller reads the router's record")
|
||||
}
|
||||
bucket, err := asksRecords(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if bucket == "" {
|
||||
return nil, errors.New("no module declares the operator channel's records")
|
||||
}
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return askIDsOf(ctx, kv, asker)
|
||||
}
|
||||
|
||||
// askIDsOf are the ids of the asks an asker has in the router's bucket.
|
||||
func askIDsOf(ctx context.Context, kv jetstream.KeyValue, asker string) ([]string, error) {
|
||||
lister, err := kv.ListKeysFiltered(ctx, asker+".>")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ids []string
|
||||
for key := range lister.Keys() {
|
||||
ids = append(ids, strings.TrimPrefix(key, asker+"."))
|
||||
}
|
||||
// The lister stops its own watch when it has sent the last key, so a stop of ours finds it stopped: nats answers that with
|
||||
// "invalid subscription", which is no failure of the read (issue 1336). Any other error stays one.
|
||||
if err := lister.Stop(); err != nil && !errors.Is(err, nats.ErrBadSubscription) {
|
||||
return nil, err
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// closedRecord is a router record of an ask that ended: it never changes, so it is read once.
|
||||
type closedRecord struct {
|
||||
ask *asks.Ask
|
||||
w *asks.Warrant
|
||||
}
|
||||
|
||||
var closedRecords sync.Map // asker + "." + id -> closedRecord
|
||||
|
||||
// closedRecordOf is the router's record of an ask, kept once it is closed. A nil ask is an ask still open, ended without a
|
||||
// choice, or not there.
|
||||
func closedRecordOf(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
if v, ok := closedRecords.Load(asker + "." + id); ok {
|
||||
r := v.(closedRecord)
|
||||
return r.ask, r.w, nil
|
||||
}
|
||||
ask, w, err := routerAskOf(ctx, inv, asker, id)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if ask != nil && w != nil {
|
||||
closedRecords.Store(asker+"."+id, closedRecord{ask, w})
|
||||
}
|
||||
return ask, w, nil
|
||||
}
|
||||
|
||||
// deliveryIDOf is the id the gate gives a delivery, and so the key of its head in the heads an Approve binds: the repository as
|
||||
// owner/name, lower-cased, `@` and the first twelve characters of the head.
|
||||
func deliveryIDOf(owner, repo, head string) string {
|
||||
if len(head) > 12 {
|
||||
head = head[:12]
|
||||
}
|
||||
return strings.ToLower(owner+"/"+repo) + "@" + head
|
||||
}
|
||||
|
||||
// coversHead says an ask the operator approved is for this merge: its Approve is bound to the heads of the pull request alone
|
||||
// (mergeAct over {delivery id: head}, which is how a pull request not in a group is asked), or, for a group's ask, whose other
|
||||
// members' heads the router's record does not carry in full, the ask as published names this pull request at this head in its
|
||||
// details ("<repo>#<number> ... at <head>:"), and the digest the warrant answers covers those words.
|
||||
func coversHead(ask *asks.Ask, owner, repo string, number int, head string) (how string, ok bool) {
|
||||
option, found := ask.Option(optionApprove)
|
||||
if !found {
|
||||
return "", false
|
||||
}
|
||||
heads := map[string]string{deliveryIDOf(owner, repo, head): head}
|
||||
if option.Performs(mergeAct(ask.ID, optionApprove, heads)) == nil {
|
||||
return "bound to the head of this pull request", true
|
||||
}
|
||||
short := head
|
||||
if len(short) > 12 {
|
||||
short = short[:12]
|
||||
}
|
||||
line := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(repo) + `#` + strconv.Itoa(number) + ` .* at ` + regexp.QuoteMeta(short) + `:`)
|
||||
if line.MatchString(ask.Details) {
|
||||
return "a group's ask naming this pull request at this head", true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// findMergeWarrant finds the warrant of a merge, with no read the controller does not make today: the router's chosen merge
|
||||
// ask for it. Under the teller (releases 3e on) the id is `merge-<group>-<heads hash>-<tail>`, which names neither the pull
|
||||
// request nor the head, so the teller's closed asks are read and the one the operator approved whose heads cover the merge's
|
||||
// second parent is the warrant (coversHead). Under mesh-delivery, which made the earlier merges, the id names the pull request
|
||||
// and the head and is found by them. Why it holds; or found false with why not, for a person to read; or an error when the
|
||||
// router could not be read, which is no answer. A variable so a test needs no bus.
|
||||
var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, repo string, number int, head string) (why string, found bool, reason string, err error) {
|
||||
reason = "the router holds no chosen merge ask of " + strings.Join(askersOfMerges(), " or ") + " that covers the head " + head[:min(12, len(head))]
|
||||
for _, asker := range askersOfMerges() {
|
||||
ids, err := routerAskIDs(ctx, inv, asker)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
for _, id := range ids {
|
||||
named := number > 0 && strings.HasPrefix(id, mergeAskPrefix(number, head))
|
||||
if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") {
|
||||
continue
|
||||
}
|
||||
ask, w, err := closedRecordOf(ctx, inv, asker, id)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
if ask == nil || w == nil {
|
||||
continue
|
||||
}
|
||||
warrant, err := caughtUpWarrant(asker, ask, w)
|
||||
if err != nil {
|
||||
if named {
|
||||
reason = fmt.Sprintf("%s's ask %s holds no operator's Approve: %v", asker, id, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
how, covers := coversHead(ask, owner, repo, number, head)
|
||||
if !covers && !named {
|
||||
continue
|
||||
}
|
||||
if !covers {
|
||||
how = "the ask's id names this pull request and head"
|
||||
}
|
||||
return warrant + ", " + how, true, "", nil
|
||||
}
|
||||
}
|
||||
return "", false, reason, nil
|
||||
}
|
||||
|
||||
// caughtTrunk is one trunk the catalogue builds from.
|
||||
type caughtTrunk struct{ Owner, Repo, Branch string }
|
||||
|
||||
func (t caughtTrunk) repository() string { return t.Owner + "/" + t.Repo }
|
||||
|
||||
// trunksOfCatalogue are the trunks of the repositories on the mesh's forge the catalogue builds a module from.
|
||||
func trunksOfCatalogue(forms *sourceForms, entries []inventory.Entry) []caughtTrunk {
|
||||
var out []caughtTrunk
|
||||
for _, e := range entries {
|
||||
if e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
owner, name, onForge := forms.onTheForge(e.Source.Repository, e.Source.Seat)
|
||||
if !onForge {
|
||||
continue
|
||||
}
|
||||
branch := followedBranch(e.Source.Ref)
|
||||
if branch == "" {
|
||||
branch = "main"
|
||||
}
|
||||
t := caughtTrunk{owner, name, branch}
|
||||
if !slices.Contains(out, t) {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// catchUpBeside runs a catch-up beside the hearing of the merge that asked for it, so the bus's acknowledgement of that merge
|
||||
// waits for no forge. A variable so a test runs it where it stands.
|
||||
var catchUpBeside = func(run func()) { go run() }
|
||||
|
||||
// catchingUp makes one catch-up at a time.
|
||||
var catchingUp sync.Mutex
|
||||
|
||||
// settledMerges are the merges this process found nothing to plan for: heard, and no record kept, because nothing reads them.
|
||||
// They are not read from the forge again.
|
||||
var settledMerges sync.Map
|
||||
|
||||
// catchUp hears the merges of the catalogue's trunks (only one, when only names it) that the controller holds no record of.
|
||||
// It says whether a trunk was left for later: the forge, the router or the store did not answer.
|
||||
func (f following) catchUp(ctx context.Context, only *caughtTrunk) (later bool) {
|
||||
catchingUp.Lock()
|
||||
defer catchingUp.Unlock()
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("catch-up: the catalogue cannot be read, so no merge is caught up now: %v\n", err)
|
||||
return true
|
||||
}
|
||||
kept, first, err := inv.EarliestKept(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("catch-up: what the controller kept cannot be read, so no merge is caught up now: %v\n", err)
|
||||
return true
|
||||
}
|
||||
if kept {
|
||||
fmt.Printf("catch-up: the controller's record of merges begins %s; a merge made before it, less %s, is history on every trunk\n",
|
||||
first.UTC().Format(time.RFC3339), mergeGrace)
|
||||
} else {
|
||||
fmt.Println("catch-up: the controller holds no merge of any trunk yet, so none can have been missed")
|
||||
}
|
||||
trunks := trunksOfCatalogue(newSourceForms(ctx, inv), entries)
|
||||
if only != nil {
|
||||
trunks = slices.DeleteFunc(trunks, func(t caughtTrunk) bool {
|
||||
return !strings.EqualFold(t.repository(), only.repository()) || t.Branch != only.Branch
|
||||
})
|
||||
}
|
||||
for _, t := range trunks {
|
||||
if f.catchUpTrunk(ctx, t, kept, first) {
|
||||
later = true
|
||||
}
|
||||
}
|
||||
return later
|
||||
}
|
||||
|
||||
func (f following) catchUpTrunk(ctx context.Context, t caughtTrunk, kept bool, first time.Time) (later bool) {
|
||||
inv := f.open.inventory
|
||||
repository := t.repository()
|
||||
merges, err := readTrunkMerges(ctx, t.Owner, t.Repo, t.Branch)
|
||||
if err != nil {
|
||||
// Not nothing to catch up: left for the next start or the next heard merge (ADR 0457 clause 3).
|
||||
fmt.Printf("catch-up: the forge did not answer for the last merges of %s %s, so they are not caught up now; the next start "+
|
||||
"or the next merge heard there does it: %v\n", repository, t.Branch, err)
|
||||
return true
|
||||
}
|
||||
// **A merge made before the controller's record of merges begins is history, never a gap** (novox/hq ADR 0463 §2).
|
||||
read := len(merges)
|
||||
merges = sinceTheRecord(merges, kept, first)
|
||||
fmt.Printf("catch-up: %s %s: %d of the last %d pull request merges were made since the record began\n", repository, t.Branch,
|
||||
len(merges), read)
|
||||
unwarranted := map[string]bool{}
|
||||
for _, tm := range merges {
|
||||
id := fmt.Sprintf("%s_%.8s", repository, tm.Commit)
|
||||
if _, done := settledMerges.Load(repository + "@" + tm.Commit); done {
|
||||
continue
|
||||
}
|
||||
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err != nil {
|
||||
fmt.Printf("catch-up: the record of %s cannot be read, so %s is not caught up now: %v\n", repository, tm.Commit[:8], err)
|
||||
return true
|
||||
} else if known {
|
||||
continue
|
||||
}
|
||||
m, err := readMergeEvent(ctx, t.Owner, t.Repo, t.Branch, tm)
|
||||
if err != nil {
|
||||
fmt.Printf("catch-up: the merge %.8s of %s cannot be read from the forge, so it is not caught up now: %v\n", tm.Commit, repository, err)
|
||||
later = true
|
||||
continue
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
facts, err := factsOfMerge(ctx, inv, m, time.Now().UTC(), entries, read)
|
||||
if err != nil {
|
||||
fmt.Printf("catch-up: %s %.8s cannot be judged now: %v\n", repository, tm.Commit, err)
|
||||
later = true
|
||||
continue
|
||||
}
|
||||
// The very test hearMerge makes: a merge it would plan, or take as a first build, is a gap; any other it would say
|
||||
// nothing reads, and is no gap for a person to decide.
|
||||
if !facts.plans() && !facts.addsModuleToBuilds {
|
||||
settledMerges.Store(repository+"@"+tm.Commit, true)
|
||||
continue
|
||||
}
|
||||
why, found, reason, err := findMergeWarrant(ctx, inv, t.Owner, t.Repo, m.Number, tm.Head)
|
||||
if err != nil {
|
||||
fmt.Printf("catch-up: the router cannot be read, so %s %.8s is not caught up now: %v\n", repository, tm.Commit, err)
|
||||
later = true
|
||||
continue
|
||||
}
|
||||
if !found {
|
||||
unwarranted[id] = true
|
||||
f.raiseMergeNotHeard(ctx, repository, t.Branch, tm, m, reason)
|
||||
continue
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
said := fmt.Sprintf("caught up: merged %s, not heard, planned %s", tm.At.Format(time.RFC3339), now.Format(time.RFC3339))
|
||||
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard; %s; on the warrant %s\n", repository, t.Branch, tm.Commit, said, why)
|
||||
m.Title = strings.TrimSpace(m.Title + " (" + said + ")")
|
||||
if err := f.hearMerge(ctx, m, now); err != nil {
|
||||
fmt.Printf("catch-up: %s %.8s could not be heard now: %v\n", repository, tm.Commit, err)
|
||||
later = true
|
||||
continue
|
||||
}
|
||||
if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err == nil && !known {
|
||||
settledMerges.Store(repository+"@"+tm.Commit, true) // heard, and nothing reads it
|
||||
}
|
||||
}
|
||||
// Cleared only when every gap was judged: one left for later may still be open.
|
||||
if !later {
|
||||
f.clearMergesHeard(ctx, repository, unwarranted)
|
||||
}
|
||||
return later
|
||||
}
|
||||
|
||||
// raiseMergeNotHeard keeps open, and observes again, the condition of a merge with no warrant to plan it. One for each
|
||||
// repository and commit, however many catch-ups find it.
|
||||
func (f following) raiseMergeNotHeard(ctx context.Context, repository, branch string, tm trunkMerge, m link.SourceMoved, reason string) {
|
||||
fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard, and is not planned: %s\n", repository, branch, tm.Commit, reason)
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
o := conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, tm.Commit), Kind: kindMergeNotHeard,
|
||||
Severity: conditions.Warning, Source: "catch-up", Resolver: conditions.ResolverOperator,
|
||||
Summary: fmt.Sprintf("the merge %.8s of %s into %s was not heard by the controller, and no operator's Approve of it is on "+
|
||||
"record to plan it on: a person decides", tm.Commit, repository, branch),
|
||||
Said: fmt.Sprintf("%s %.8s (pull request %d, head %.8s, merged %s): %s", repository, tm.Commit, m.Number, tm.Head,
|
||||
tm.At.Format(time.RFC3339), reason),
|
||||
Headline: fmt.Sprintf("A merge into %s was never delivered", repository),
|
||||
Explanation: fmt.Sprintf("A merge into %s was made while the controller did not hear it, and no Approve of yours for it is on "+
|
||||
"record that the controller can plan it on. It stays open while the merge is among the latest %d merges of that repository. "+
|
||||
"The details say what was looked for.", repository, catchUpWindow),
|
||||
Needs: "decide whether that merge should be delivered, and say so if it should.",
|
||||
Resolved: "the merge is planned, or is no longer among the latest merges of its repository"}
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
fmt.Printf("catch-up: the condition of %s %.8s could not be kept: %v\n", repository, tm.Commit, err)
|
||||
}
|
||||
}
|
||||
|
||||
// clearMergesHeard clears the open conditions of a repository's merges not heard that this catch-up did not find again: the
|
||||
// gap was planned, or left the window.
|
||||
func (f following) clearMergesHeard(ctx context.Context, repository string, still map[string]bool) {
|
||||
if conditionsFrom == nil {
|
||||
return
|
||||
}
|
||||
open, err := conditionsFrom.Open(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("catch-up: the open conditions cannot be read, so none is cleared now: %v\n", err)
|
||||
return
|
||||
}
|
||||
for _, c := range open {
|
||||
if c.Kind != kindMergeNotHeard || c.Subject.Scope != conditions.ScopeMerge ||
|
||||
!strings.HasPrefix(c.Subject.ID, repository+"_") || still[c.Subject.ID] {
|
||||
continue
|
||||
}
|
||||
if _, err := conditionsFrom.Clear(ctx, c.Key, "the merge is planned, or left the last merges of its trunk"); err != nil {
|
||||
fmt.Printf("catch-up: the condition %s could not be cleared: %v\n", c.Key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catchUpAtStart catches up every trunk once the controller serves, and again every minute while a trunk was left for later.
|
||||
func catchUpAtStart(ctx context.Context, open *stores) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(30 * time.Second):
|
||||
}
|
||||
for try := 0; try < 20; try++ {
|
||||
if !(following{open: open}).catchUp(ctx, nil) {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Minute):
|
||||
}
|
||||
}
|
||||
fmt.Println("catch-up: a trunk is still left for later after twenty tries; the next merge heard there does it")
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear.
|
||||
|
||||
// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the
|
||||
// router holds an operator's Approve for.
|
||||
type aTrunk struct {
|
||||
merges []trunkMerge
|
||||
module map[string]string
|
||||
approved map[string]bool
|
||||
read []string // the trunks asked of the forge, as owner/repo
|
||||
events int // merges read in full from the forge
|
||||
silent bool // the forge does not answer
|
||||
}
|
||||
|
||||
func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk {
|
||||
t.Helper()
|
||||
tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}}
|
||||
for i, m := range modules {
|
||||
c := fmt.Sprintf("%s-c%02d", repo, i+1)
|
||||
tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1),
|
||||
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c),
|
||||
At: t0.Add(time.Duration(i) * time.Minute)})
|
||||
tr.module[c] = m
|
||||
tr.approved[c] = true
|
||||
}
|
||||
was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant
|
||||
trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) {
|
||||
tr.read = append(tr.read, owner+"/"+name)
|
||||
if tr.silent {
|
||||
return nil, errors.New("the forge did not answer")
|
||||
}
|
||||
if page > 1 || owner+"/"+name != "novox/"+repo {
|
||||
return nil, nil
|
||||
}
|
||||
var newest []trunkCommit
|
||||
for i := len(tr.merges) - 1; i >= 0; i-- {
|
||||
m := tr.merges[i]
|
||||
newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At})
|
||||
}
|
||||
return newest, nil
|
||||
}
|
||||
readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
||||
tr.events++
|
||||
m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At)
|
||||
m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head
|
||||
_, m.Number, _ = pullOfSubject(tm.Subject)
|
||||
return m, nil
|
||||
}
|
||||
findMergeWarrant = func(_ context.Context, _ catalogueReader, _, _ string, number int, head string) (string, bool, string, error) {
|
||||
for _, m := range tr.merges {
|
||||
if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] {
|
||||
return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil
|
||||
}
|
||||
}
|
||||
return "", false, "the router holds no chosen merge ask for the pull request and head", nil
|
||||
}
|
||||
wasBeside := catchUpBeside
|
||||
catchUpBeside = func(run func()) { run() }
|
||||
t.Cleanup(func() {
|
||||
trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside
|
||||
settledMerges.Clear()
|
||||
})
|
||||
return tr
|
||||
}
|
||||
|
||||
// aKeptMergeBefore is a merge the controller heard before the merges of a test: where its record begins, so what a trunk holds from
|
||||
// then on can be a gap (novox/hq ADR 0463 §2).
|
||||
func aKeptMergeBefore(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
hear(t, open, catalogueMerge("mesh-catalog-seed", "app", t0.Add(-time.Hour)), t0.Add(-time.Hour))
|
||||
}
|
||||
|
||||
func mergeKnown(t *testing.T, open *stores, commit string) bool {
|
||||
t.Helper()
|
||||
_, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return known
|
||||
}
|
||||
|
||||
// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said
|
||||
// as caught up; a second start plans nothing new.
|
||||
func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
|
||||
hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second))
|
||||
hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second))
|
||||
if mergeKnown(t, open, "mesh-catalog-c02") {
|
||||
t.Fatal("the merge to be caught up is known before the catch-up")
|
||||
}
|
||||
if (following{open: open}).catchUp(t.Context(), nil) {
|
||||
t.Fatal("the catch-up says a trunk was left for later")
|
||||
}
|
||||
if !mergeKnown(t, open, "mesh-catalog-c02") {
|
||||
t.Fatal("the merge the controller did not hear was not planned")
|
||||
}
|
||||
kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02")
|
||||
if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") {
|
||||
t.Fatalf("the merge does not say it was caught up: %q", said)
|
||||
}
|
||||
cutAt(t, open, t0.Add(10*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) == 0 {
|
||||
t.Fatal("no walk answers the caught-up merge")
|
||||
}
|
||||
if _, in := ws[0].Modules["notes"]; !in {
|
||||
t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules)
|
||||
}
|
||||
before := tr.events
|
||||
if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before {
|
||||
t.Fatalf("a second start read %d merge(s) in full again", tr.events-before)
|
||||
}
|
||||
}
|
||||
|
||||
// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the
|
||||
// earlier first, and the forge is read for that repository's trunk only.
|
||||
func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app")
|
||||
var order []string
|
||||
inner := readMergeEvent
|
||||
readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) {
|
||||
order = append(order, tm.Commit)
|
||||
return inner(ctx, owner, name, branch, tm)
|
||||
}
|
||||
t.Cleanup(func() { readMergeEvent = inner })
|
||||
tr.read = nil
|
||||
heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute))
|
||||
if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} {
|
||||
if !mergeKnown(t, open, c) {
|
||||
t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c)
|
||||
}
|
||||
}
|
||||
if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" {
|
||||
t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order)
|
||||
}
|
||||
if len(tr.read) == 0 {
|
||||
t.Fatal("the forge was not read for the trunk of the merge heard")
|
||||
}
|
||||
for _, r := range tr.read {
|
||||
if r != "novox/mesh-catalog" {
|
||||
t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window.
|
||||
func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
modules := make([]string, 51)
|
||||
for i := range modules {
|
||||
modules[i] = "app"
|
||||
}
|
||||
aTrunkOf(t, "mesh-catalog", modules...)
|
||||
// Fifty are known; the oldest is not.
|
||||
for i := 2; i <= 51; i++ {
|
||||
c := fmt.Sprintf("mesh-catalog-c%02d", i)
|
||||
hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second))
|
||||
}
|
||||
// Paged as the forge pages: fifty a page, and a trunk of another repository is empty (a short page, or none).
|
||||
pages := trunkCommitsPage
|
||||
trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) {
|
||||
all, err := pages(ctx, owner, name, branch, 1)
|
||||
if err != nil || page < 1 || (page-1)*50 >= len(all) {
|
||||
return nil, err
|
||||
}
|
||||
return all[(page-1)*50 : min(page*50, len(all))], nil
|
||||
}
|
||||
t.Cleanup(func() { trunkCommitsPage = pages })
|
||||
(following{open: open}).catchUp(t.Context(), nil)
|
||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatal("the oldest of fifty-one merges was read and planned, outside the window")
|
||||
}
|
||||
}
|
||||
|
||||
// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and
|
||||
// commit; a second start leaves that one condition open, observed twice; planning the gap clears it.
|
||||
func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
aKeptMergeBefore(t, open)
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
tr := aTrunkOf(t, "mesh-catalog", "notes")
|
||||
tr.approved["mesh-catalog-c01"] = false
|
||||
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
|
||||
catch()
|
||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatal("a merge with no Approve on record was planned")
|
||||
}
|
||||
raised := func() []conditions.Condition {
|
||||
t.Helper()
|
||||
all, err := keeper.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []conditions.Condition
|
||||
for _, c := range all {
|
||||
if c.Kind == kindMergeNotHeard {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
one := raised()
|
||||
if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") {
|
||||
t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one)
|
||||
}
|
||||
catch()
|
||||
if again := raised(); len(again) != 1 || again[0].Observations < 2 {
|
||||
t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again)
|
||||
}
|
||||
tr.approved["mesh-catalog-c01"] = true
|
||||
catch()
|
||||
if !mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatal("the gap was not planned once its Approve was on record")
|
||||
}
|
||||
if left := raised(); len(left) != 0 {
|
||||
t.Fatalf("planning the gap left its condition open: %+v", left)
|
||||
}
|
||||
}
|
||||
|
||||
// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it.
|
||||
func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
aKeptMergeBefore(t, open)
|
||||
tr := aTrunkOf(t, "mesh-catalog", "notes")
|
||||
tr.silent = true
|
||||
if later := (following{open: open}).catchUp(t.Context(), nil); !later {
|
||||
t.Fatal("a forge that did not answer counted as nothing to catch up")
|
||||
}
|
||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatal("a merge was planned without the forge's word")
|
||||
}
|
||||
tr.silent = false
|
||||
if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatalf("the next start did not catch up (left for later: %v)", later)
|
||||
}
|
||||
}
|
||||
|
||||
// Finding 2 of the review: a gap that adds a module is a gap, judged as hearMerge judges it. With the operator's Approve on record it
|
||||
// is heard (hearMerge asks the new module in its walk); without, it raises the condition and is not lost in silence.
|
||||
func TestAGapThatAddsAModuleIsHeardOnItsWarrantAndRaisedWithout(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
aKeptMergeBefore(t, open)
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
tr := aTrunkOf(t, "mesh-catalog", "newmodule")
|
||||
tr.approved["mesh-catalog-c01"] = false
|
||||
catch := func() { (following{open: open}).catchUp(t.Context(), nil) }
|
||||
catch()
|
||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatal("a merge adding a module was heard with no Approve on record")
|
||||
}
|
||||
all, err := keeper.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raised int
|
||||
for _, c := range all {
|
||||
if c.Kind == kindMergeNotHeard {
|
||||
raised++
|
||||
}
|
||||
}
|
||||
if raised != 1 {
|
||||
t.Fatalf("a merge adding a module with no Approve raised %d condition(s), not one: it was lost in silence", raised)
|
||||
}
|
||||
tr.approved["mesh-catalog-c01"] = true
|
||||
catch()
|
||||
if !mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatal("a merge adding a module was not heard once its Approve was on record")
|
||||
}
|
||||
}
|
||||
|
||||
// A trunk with merges from before the controller began keeping merges, and merges since: only the later ones are gaps.
|
||||
func TestAMergeFromBeforeTheRecordBeganIsHistoryAndNoCondition(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes", "notes")
|
||||
tr.approved["mesh-catalog-c01"], tr.approved["mesh-catalog-c02"], tr.approved["mesh-catalog-c03"] = false, false, false
|
||||
// The record begins at the second merge: the first is two days older.
|
||||
tr.merges[0].At = t0.Add(-48 * time.Hour)
|
||||
hear(t, open, catalogueMerge("mesh-catalog-c02", "notes", t0.Add(time.Minute)), t0.Add(time.Minute))
|
||||
delete(tr.approved, "mesh-catalog-c02")
|
||||
(following{open: open}).catchUp(t.Context(), nil)
|
||||
// The first merge (two days before the record) was not even read from the forge: only the third, which is unknown, was.
|
||||
if tr.events != 1 {
|
||||
t.Fatalf("%d merge(s) were read from the forge: the one from before the record is history and is not read", tr.events)
|
||||
}
|
||||
all, err := keeper.Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raised int
|
||||
for _, c := range all {
|
||||
if c.Kind == kindMergeNotHeard {
|
||||
raised++
|
||||
}
|
||||
}
|
||||
if raised != 1 {
|
||||
t.Fatalf("%d condition(s) raised: only the merge since the record began (the third) is a gap", raised)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0463 §2: the record begins at the earliest merge kept on ANY trunk. A trunk the controller kept no merge of has the
|
||||
// merges made since as gaps (mesh-delivery #1 on novox), and the merges made before as history.
|
||||
func TestATrunkWithNoKeptMergeHasItsLaterMergesAsGapsAndItsEarlierOnesAsHistory(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksRecorded(t)
|
||||
// The record begins on another trunk, at t0 less an hour.
|
||||
hear(t, open, repoMerge("one", "c-one", t0.Add(-time.Hour)), t0.Add(-time.Hour))
|
||||
tr := aTrunkOf(t, "mesh-catalog", "notes", "notes")
|
||||
tr.merges[0].At = t0.Add(-72 * time.Hour) // before the record: history
|
||||
(following{open: open}).catchUp(t.Context(), nil)
|
||||
if mergeKnown(t, open, "mesh-catalog-c01") {
|
||||
t.Fatal("a merge from before the record began was heard")
|
||||
}
|
||||
if !mergeKnown(t, open, "mesh-catalog-c02") {
|
||||
t.Fatal("a merge since the record began, on a trunk with no merge kept, was not caught up")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The pieces of the catch-up that need no store (novox/hq ADR 0457, issue 1336).
|
||||
|
||||
func TestAMergeCommitsSubjectNamesItsPullRequest(t *testing.T) {
|
||||
title, number, ok := pullOfSubject("Merge pull request 'Fix keycloak's sign-in (issue 1332)' (#378) from fix/1332-sign-in into main")
|
||||
if !ok || number != 378 || title != "Fix keycloak's sign-in (issue 1332)" {
|
||||
t.Fatalf("read %q %d %v", title, number, ok)
|
||||
}
|
||||
if b := headBranchOfSubject("Merge pull request 'x' (#378) from fix/1332-sign-in into main"); b != "fix/1332-sign-in" {
|
||||
t.Fatalf("the head branch read as %q", b)
|
||||
}
|
||||
if _, _, ok := pullOfSubject("Merge branch 'main' of somewhere"); ok {
|
||||
t.Fatal("a merge that names no pull request read as one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheLastFiftyMergesOfATrunkAreRead(t *testing.T) {
|
||||
pages := map[int][]trunkCommit{}
|
||||
var all []trunkCommit // newest first: 51 merges, a plain commit between each pair
|
||||
for i := 51; i >= 1; i-- {
|
||||
all = append(all, trunkCommit{Commit: fmt.Sprintf("merge-%02d", i), SecondParent: fmt.Sprintf("head-%02d", i),
|
||||
Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from fix/x into main", i, i)},
|
||||
trunkCommit{Commit: fmt.Sprintf("plain-%02d", i)})
|
||||
}
|
||||
for i := 0; i < len(all); i += 50 {
|
||||
pages[i/50+1] = all[i:min(i+50, len(all))]
|
||||
}
|
||||
was := trunkCommitsPage
|
||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { return pages[page], nil }
|
||||
t.Cleanup(func() { trunkCommitsPage = was })
|
||||
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != catchUpWindow || got[0].Commit != "merge-02" || got[len(got)-1].Commit != "merge-51" {
|
||||
t.Fatalf("read %d merges from %s to %s: the oldest of 51 is outside the window", len(got), got[0].Commit, got[len(got)-1].Commit)
|
||||
}
|
||||
if got[0].Head != "head-02" {
|
||||
t.Fatalf("a merge's head is its second parent, not %q", got[0].Head)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) {
|
||||
was := trunkCommitsPage
|
||||
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) {
|
||||
return nil, errors.New("the forge did not answer")
|
||||
}
|
||||
t.Cleanup(func() { trunkCommitsPage = was })
|
||||
if got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main"); err == nil {
|
||||
t.Fatalf("a forge that did not answer read as %d merges: nothing to catch up", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) {
|
||||
ok := approvedMerge(t)
|
||||
if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil {
|
||||
t.Errorf("the operator's Approve under the teller was refused: %v", err)
|
||||
}
|
||||
// 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken.
|
||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil {
|
||||
t.Error("a warrant under mesh-delivery was accepted")
|
||||
}
|
||||
if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil {
|
||||
t.Error("a warrant of an asker the controller takes none from was accepted")
|
||||
}
|
||||
declined := ok.warrant
|
||||
declined.Option, declined.Label = "decline", "Decline"
|
||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &declined); err == nil {
|
||||
t.Error("a Decline was accepted as a warrant")
|
||||
}
|
||||
unproved := ok.warrant
|
||||
by := *unproved.By
|
||||
by.Verified = ""
|
||||
unproved.By = &by
|
||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &unproved); err == nil {
|
||||
t.Error("an answer whose channel proves nobody was accepted")
|
||||
}
|
||||
if _, err := caughtUpWarrant(theDelivery, nil, nil); err == nil {
|
||||
t.Error("a router that holds nothing was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// aTellersAsk is the ask the delivery gate composes for a pull request, as the router holds it once the operator chose Approve
|
||||
// (the gate's composeMergeAsk: its id from sdkdelivery.AskingID of the group and the heads, its Approve bound to mergeAct over the
|
||||
// heads, the pull request and its head's first twelve digits in the details), with the warrant the router wrote.
|
||||
func aTellersAsk(t *testing.T, group string, members ...sdkdelivery.Head) (asks.Ask, asks.Warrant) {
|
||||
t.Helper()
|
||||
key, err := sdkdelivery.MergeAskKey(group, members)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := sdkdelivery.AskingID(key, 1)
|
||||
heads := map[string]string{}
|
||||
var details []string
|
||||
for _, m := range members {
|
||||
owner, repo, _ := strings.Cut(m.Repository, "/")
|
||||
heads[deliveryIDOf(owner, repo, m.Head)] = m.Head
|
||||
details = append(details, fmt.Sprintf("%s#%d %q at %s: issues: none named; review: approved; reaches: no delivery plan yet", repo, m.Number, "title", m.Head[:12]))
|
||||
}
|
||||
bind := func(option string) string {
|
||||
d, err := asks.ActDigest(mergeAct(id, option, heads))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
at := time.Date(2026, 10, 20, 12, 2, 1, 0, time.UTC)
|
||||
ask := asks.Ask{ID: id, Headline: "Merge into main", Explanation: "passed its checks.", Who: asks.Operator, Expires: at.Add(24 * time.Hour),
|
||||
OnExpiry: "nothing is merged; the ask is made again", Details: strings.Join(details, "\n"),
|
||||
Options: []asks.Option{
|
||||
{ID: optionApprove, Label: "Approve", Does: "merges", Level: asks.Approve, Binds: bind(optionApprove)},
|
||||
{ID: "decline", Label: "Decline", Does: "nothing is merged", Level: asks.Approve, Binds: bind("decline")}}}
|
||||
w := asks.Warrant{Ask: id, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve",
|
||||
Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"},
|
||||
At: at.Add(time.Minute), AskDigest: ask.Digest()}
|
||||
return ask, w
|
||||
}
|
||||
|
||||
// withRouter hands findMergeWarrant these asks as the router's records, by asker.
|
||||
func withRouter(t *testing.T, records map[string]map[string]struct {
|
||||
ask asks.Ask
|
||||
w asks.Warrant
|
||||
}) {
|
||||
t.Helper()
|
||||
wasIDs, wasAsk := routerAskIDs, routerAskOf
|
||||
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
|
||||
routerAskIDs = func(_ context.Context, _ catalogueReader, asker string) ([]string, error) {
|
||||
var ids []string
|
||||
for id := range records[asker] {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
r, ok := records[asker][id]
|
||||
if !ok {
|
||||
return nil, nil, nil
|
||||
}
|
||||
return &r.ask, &r.w, nil
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
routerAskIDs, routerAskOf = wasIDs, wasAsk
|
||||
closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true })
|
||||
})
|
||||
}
|
||||
|
||||
type aRouterRecord = struct {
|
||||
ask asks.Ask
|
||||
w asks.Warrant
|
||||
}
|
||||
|
||||
// The teller's asks, as the router held them on 2026-10-20: the id is the router's own key, rebuilt here from the group and the
|
||||
// head the gate composed it for (mesh-controller#405 at 8a330afd, mesh-catalog#381 at 1c702422, mesh-host#124 at 52600ec7). It names
|
||||
// neither the pull request nor the head, so the warrant is found by the heads the Approve is bound to.
|
||||
func TestATellersWarrantIsFoundByTheHeadItsApproveIsBoundTo(t *testing.T) {
|
||||
real := []struct {
|
||||
group, key, owner, repo string
|
||||
number int
|
||||
head string
|
||||
}{
|
||||
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "novox", "mesh-controller", 405, "8a330afd44ba852873339cf1210e90989dca46e1"},
|
||||
{"fix/1351-remove-todays-delivery-module", "merge-fix_1351_remove_todays_delivery_module-64a353-d93b97b9697b", "novox", "mesh-catalog", 381, "1c7024224dd77c5b603299378883e0993b2e2fdd"},
|
||||
{"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-6a59ba48da14-424e053ebc9c", "novox", "mesh-host", 124, "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"},
|
||||
}
|
||||
teller := map[string]aRouterRecord{}
|
||||
for _, r := range real {
|
||||
ask, w := aTellersAsk(t, r.group, sdkdelivery.Head{Repository: r.owner + "/" + r.repo, Number: r.number, Head: r.head})
|
||||
if ask.ID != r.key {
|
||||
t.Fatalf("the id of %s#%d is %q, and the router held it as %q: the fixture is not the router's", r.repo, r.number, ask.ID, r.key)
|
||||
}
|
||||
teller[ask.ID] = aRouterRecord{ask, w}
|
||||
}
|
||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: teller})
|
||||
for _, r := range real {
|
||||
why, found, reason, err := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, r.head)
|
||||
if err != nil || !found || !strings.Contains(why, r.key) || !strings.Contains(why, "bound to the head of this pull request") {
|
||||
t.Errorf("%s#%d: the warrant was not found: %q %v %q %v", r.repo, r.number, why, found, reason, err)
|
||||
}
|
||||
// Another head of the same pull request is another merge: the Approve was not given for it.
|
||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, "ffffffffffffffffffffffffffffffffffffffff"); found {
|
||||
t.Errorf("%s#%d: another head found the warrant of this one", r.repo, r.number)
|
||||
}
|
||||
}
|
||||
// One pull request's Approve is none for another repository's merge at the same head digits.
|
||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-tools", 405, real[0].head); found {
|
||||
t.Error("a merge of another repository found mesh-controller#405's warrant")
|
||||
}
|
||||
}
|
||||
|
||||
// A group's ask binds several heads, and the router's record carries only twelve digits of the other members': the pull request is
|
||||
// found in the ask as it was published, which the warrant's digest covers.
|
||||
func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testing.T) {
|
||||
a := sdkdelivery.Head{Repository: "novox/mesh-host", Number: 124, Head: "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"}
|
||||
b := sdkdelivery.Head{Repository: "novox/mesh-controller", Number: 405, Head: "8a330afd44ba852873339cf1210e90989dca46e1"}
|
||||
ask, w := aTellersAsk(t, "feat/r4b-sdk-users", a, b)
|
||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, w}}})
|
||||
for _, h := range []sdkdelivery.Head{a, b} {
|
||||
owner, repo, _ := strings.Cut(h.Repository, "/")
|
||||
why, found, reason, err := findMergeWarrant(t.Context(), nil, owner, repo, h.Number, h.Head)
|
||||
if err != nil || !found || !strings.Contains(why, "a group's ask naming this pull request at this head") {
|
||||
t.Errorf("%s#%d: not found in the group's ask: %q %v %q %v", repo, h.Number, why, found, reason, err)
|
||||
}
|
||||
}
|
||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 381, a.Head); found {
|
||||
t.Error("a pull request the group's ask does not name found its warrant")
|
||||
}
|
||||
// A Decline is no warrant.
|
||||
declined := w
|
||||
declined.Option, declined.Label = "decline", "Decline"
|
||||
withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, declined}}})
|
||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-host", 124, a.Head); found {
|
||||
t.Error("a Decline found a warrant")
|
||||
}
|
||||
}
|
||||
|
||||
// A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do.
|
||||
func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) {
|
||||
was := trunkCommitsPage
|
||||
t.Cleanup(func() { trunkCommitsPage = was })
|
||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
|
||||
if page > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
return []trunkCommit{{Commit: "m2", SecondParent: "h2", Subject: "Merge pull request 'two' (#2) from fix/b into main"},
|
||||
{Commit: "p"}, {Commit: "m1", SecondParent: "h1", Subject: "Merge pull request 'one' (#1) from fix/a into main"}}, nil
|
||||
}
|
||||
got, err := readTrunkMerges(t.Context(), "novox", "x", "main")
|
||||
if err != nil || len(got) != 2 || got[0].Commit != "m1" || got[1].Commit != "m2" {
|
||||
t.Fatalf("a short trunk read as %+v %v", got, err)
|
||||
}
|
||||
trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { return nil, nil }
|
||||
if got, err := readTrunkMerges(t.Context(), "novox", "x", "main"); err != nil || len(got) != 0 {
|
||||
t.Fatalf("an empty trunk read as %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0253: what the operator reads of a merge not heard is plain: no hash, no identifier, nothing to do "by hand".
|
||||
func TestTheConditionOfAMergeNotHeardIsInPlainWords(t *testing.T) {
|
||||
keeper, _ := withConditionsInMemory(t)
|
||||
tm := trunkMerge{Commit: "c0880c4d1f2e3a4b5c6d7e8f9a0b1c2d3e4f5a6b", Head: "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372"}
|
||||
(following{}).raiseMergeNotHeard(t.Context(), "novox/mesh-catalog", "main", tm, link.SourceMoved{Number: 378},
|
||||
"the router holds no chosen merge ask of mesh-delivery-teller or mesh-delivery that covers the head b9104454f9a5")
|
||||
open, err := keeper.Open(t.Context())
|
||||
if err != nil || len(open) != 1 {
|
||||
t.Fatalf("opened %d conditions, %v", len(open), err)
|
||||
}
|
||||
c := open[0]
|
||||
if why, ok := conditions.PlainWords(conditions.Words{Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs,
|
||||
Resolved: c.Resolved}); !ok {
|
||||
t.Fatalf("the condition's words are not plain: %s\n%q\n%q\n%q", why, c.Headline, c.Explanation, c.Needs)
|
||||
}
|
||||
}
|
||||
|
||||
// The trunk's history lists the merges made inside a pull request's branch too. On novox (2026-10-20) 15 of the 15 non-pull-request
|
||||
// "gaps" of mesh-catalog and mesh-controller were such merges; only a pull request's merge into this trunk is a merge to catch up.
|
||||
func TestAMergeMadeInsideABranchIsNoMergeOfTheTrunk(t *testing.T) {
|
||||
was := trunkCommitsPage
|
||||
t.Cleanup(func() { trunkCommitsPage = was })
|
||||
trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) {
|
||||
if page > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
return []trunkCommit{
|
||||
{Commit: "c0880c4d", SecondParent: "b9104454", Subject: "Merge pull request 'Read the grant file' (#378) from fix/1332 into main"},
|
||||
{Commit: "6394a62a", SecondParent: "x1", Subject: "Merge remote-tracking branch 'origin/main' into fix/1081-withdraw"},
|
||||
{Commit: "896f1d28", SecondParent: "x2", Subject: "Merge feat/agents-write-git-ssh into feat/agents-write-git-ssh-f (issue 955 F)"},
|
||||
{Commit: "4d03755c", SecondParent: "x3", Subject: "Bring the branch up to main after #354 (issue 1047)"},
|
||||
{Commit: "other", SecondParent: "x4", Subject: "Merge pull request 'Elsewhere' (#5) from fix/y into nox-mesh"},
|
||||
}, nil
|
||||
}
|
||||
got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main")
|
||||
if err != nil || len(got) != 1 || got[0].Commit != "c0880c4d" {
|
||||
t.Fatalf("read %+v %v: only keycloak's pull request merge into main is a merge to catch up", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0463 §2: the record of merges begins at the earliest merge the controller holds on any trunk. A merge before it is
|
||||
// history on every trunk; a trunk with no merge kept of its own still has the merges made since as gaps (mesh-delivery #1 on novox).
|
||||
func TestOnlyTheMergesSinceTheRecordBeganAreGapsOnEveryTrunk(t *testing.T) {
|
||||
first := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) // the first merge it heard, on some trunk of the mesh
|
||||
trunk := func(prefix string) []trunkMerge {
|
||||
return []trunkMerge{
|
||||
{Commit: prefix + "-old-1", At: first.Add(-30 * 24 * time.Hour)},
|
||||
{Commit: prefix + "-old-2", At: first.Add(-2 * time.Hour)},
|
||||
{Commit: prefix + "-within-grace", At: first.Add(-mergeGrace / 2)},
|
||||
{Commit: prefix + "-since", At: first.Add(24 * time.Hour)},
|
||||
}
|
||||
}
|
||||
// Two trunks, one of which the controller has kept no merge of: both read by the one starting point.
|
||||
for _, prefix := range []string{"kept-trunk", "trunk-with-no-kept-merge"} {
|
||||
var commits []string
|
||||
for _, m := range sinceTheRecord(trunk(prefix), true, first) {
|
||||
commits = append(commits, m.Commit)
|
||||
}
|
||||
if want := prefix + "-within-grace " + prefix + "-since"; strings.Join(commits, " ") != want {
|
||||
t.Errorf("%s: the gaps are %v, want %q: earlier ones are history on every trunk, later ones are gaps", prefix, commits, want)
|
||||
}
|
||||
}
|
||||
if none := sinceTheRecord(trunk("x"), false, time.Time{}); len(none) != 0 {
|
||||
t.Errorf("with no merge kept anywhere there are %d gap(s), but no record to have one in", len(none))
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,7 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
||||
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||
go copySentBuildsSoon(ctx, inv)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -61,6 +62,39 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||
|
||||
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
||||
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
||||
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
||||
// that names a commit and empties each column it has copied.
|
||||
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
failed := false
|
||||
for {
|
||||
_, err := inv.CopySentBuilds(ctx)
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
||||
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
||||
if err != nil {
|
||||
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chainMoveRetry is how long the copy waits before it tries again.
|
||||
const chainMoveRetry = 30 * time.Second
|
||||
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there,
|
||||
// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live
|
||||
// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's
|
||||
// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send
|
||||
// is recorded as the controller records one it made, and the gate's facts are the store's.
|
||||
func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var sent [][]string
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
digest := fmt.Sprintf("d-%s-%d", node, len(sent))
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest,
|
||||
map[string]string{"notes": current["notes"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
// A merge of notes is heard, and its window closes: the batch is cut into the walk.
|
||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
||||
}
|
||||
rollout := ws[0].ID
|
||||
|
||||
// The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the
|
||||
// outcome in as it does.
|
||||
asking, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ {
|
||||
advancePlans(ctx, open)
|
||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
buildID := asking.Modules["notes"].Build
|
||||
if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) {
|
||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
||||
}
|
||||
|
||||
// The walk sends the tier to its first machine.
|
||||
for try := 0; try < 3 && len(sent) == 0; try++ {
|
||||
advancePlans(ctx, open)
|
||||
}
|
||||
if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) {
|
||||
t.Fatalf("sent %v", sent)
|
||||
}
|
||||
|
||||
// One read of the assignment.
|
||||
on, err := inv.AssignmentsOn(ctx, "anchor")
|
||||
a, ok := assignmentOf(on, "notes")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("the assignments of anchor: %+v %v", on, err)
|
||||
}
|
||||
// Beside it a module pulled in by the assignment, which no send of this walk put anywhere.
|
||||
if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") {
|
||||
t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard)
|
||||
}
|
||||
if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout ||
|
||||
a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() {
|
||||
t.Fatalf("the assignment names %+v, not commit c-notes, build %s, rollout %s", a, buildID, rollout)
|
||||
}
|
||||
// One read of the rollout.
|
||||
p, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}}
|
||||
if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 ||
|
||||
p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil {
|
||||
t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange)
|
||||
}
|
||||
// And one read of the build names the same commit.
|
||||
built, found, err := inv.BuildByID(ctx, buildID)
|
||||
if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" {
|
||||
t.Fatalf("the build: %+v %v %v", built, found, err)
|
||||
}
|
||||
// The verbs say the links.
|
||||
if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") ||
|
||||
!strings.Contains(lines[1], buildID) {
|
||||
t.Fatalf("`plans` says %v", lines)
|
||||
}
|
||||
if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) {
|
||||
t.Fatalf("`node show` says %v", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks
|
||||
// nothing (issue 1095).
|
||||
func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
node := nodeID(t, open, "anchor")
|
||||
inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}})
|
||||
if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent again by hand, with no rollout: the assignment still names the rollout that put the build there.
|
||||
if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
on, _ := inv.AssignmentsOn(ctx, "anchor")
|
||||
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
||||
t.Fatalf("the assignment names %+v", on)
|
||||
}
|
||||
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
|
||||
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
|
||||
t.Fatalf("a send of unknown builds left %+v", on)
|
||||
}
|
||||
// A module sent a commit no build is kept of is named by its commit alone.
|
||||
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" ||
|
||||
runOf(on, "notes").Rollout != "" {
|
||||
t.Fatalf("the assignment after a push of c2 by hand names %+v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it.
|
||||
func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) {
|
||||
for _, a := range on {
|
||||
if a.Module == module {
|
||||
return a, true
|
||||
}
|
||||
}
|
||||
return chain.Assignment{}, false
|
||||
}
|
||||
|
||||
func runOf(on []chain.Assignment, module string) chain.Assignment {
|
||||
a, _ := assignmentOf(on, module)
|
||||
return a
|
||||
}
|
||||
|
||||
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
|
||||
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
|
||||
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
|
||||
// and the table of pending assignments is read by nobody. No key is written by hand.
|
||||
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatalf("an assignment while its build runs was refused: %v", err)
|
||||
}
|
||||
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
|
||||
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
|
||||
!strings.HasPrefix(waiting.Pending.ID, "p-") {
|
||||
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
|
||||
}
|
||||
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
|
||||
t.Fatalf("a module only waiting is assigned: %v", got)
|
||||
}
|
||||
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
|
||||
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
|
||||
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
|
||||
}
|
||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("the build registered sensors and it was not assigned")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a
|
||||
// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of
|
||||
// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the
|
||||
// build before and after.
|
||||
|
||||
// rolloutModules is what a plan sends of each module: the commit and the build of it.
|
||||
func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule {
|
||||
out := make(map[string]inventory.RolloutModule, len(p.Modules))
|
||||
for m, s := range p.Modules {
|
||||
if s != nil && s.Commit != "" {
|
||||
out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout.
|
||||
func sendingAs(ctx context.Context, p *inventory.Plan) context.Context {
|
||||
return inventory.WithRollout(ctx, p.ID, rolloutModules(p))
|
||||
}
|
||||
|
||||
// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit,
|
||||
// else the commit the machine was last sent for it. Read before the send, which writes the new ones.
|
||||
func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref {
|
||||
was := map[string]chain.Ref{}
|
||||
if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known {
|
||||
for m, commit := range sent {
|
||||
was[m] = chain.Ref{Commit: commit}
|
||||
}
|
||||
}
|
||||
if ran, err := inv.AssignmentRuns(ctx, node); err == nil {
|
||||
for m, r := range ran {
|
||||
was[m] = r
|
||||
}
|
||||
}
|
||||
return was
|
||||
}
|
||||
|
||||
// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and
|
||||
// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the
|
||||
// module is replaced, so a send made again says it once.
|
||||
func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) {
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}}
|
||||
if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 {
|
||||
change.Before = p.Changed[at].Before
|
||||
p.Changed[at] = change
|
||||
} else {
|
||||
p.Changed = append(p.Changed, change)
|
||||
}
|
||||
if s.Build != "" && !slices.Contains(p.Builds, s.Build) {
|
||||
p.Builds = append(p.Builds, s.Build)
|
||||
}
|
||||
}
|
||||
if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) {
|
||||
sent := at.UTC()
|
||||
p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent})
|
||||
}
|
||||
}
|
||||
|
||||
// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none.
|
||||
func refSaid(r chain.Ref) string {
|
||||
switch {
|
||||
case r.Build == "" && r.Commit == "":
|
||||
return "nothing"
|
||||
case r.Build == "":
|
||||
return "commit " + short(r.Commit)
|
||||
}
|
||||
return r.Build + " (" + short(r.Commit) + ")"
|
||||
}
|
||||
|
||||
// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after.
|
||||
func changedLines(p inventory.Plan) []string {
|
||||
if len(p.Changed) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := []string{" changed:"}
|
||||
for _, c := range p.Changed {
|
||||
out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the
|
||||
// rollout that put it there and since when.
|
||||
func runLines(assigned []chain.Assignment) []string {
|
||||
var out []string
|
||||
for _, a := range assigned {
|
||||
if a.Build == "" && a.Commit == "" {
|
||||
continue
|
||||
}
|
||||
line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit}))
|
||||
if a.Rollout != "" {
|
||||
line += ", sent by " + a.Rollout
|
||||
}
|
||||
if !a.Since.IsZero() {
|
||||
line += " at " + a.Since.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]string{" what they run, as the chain names it:"}, out...)
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one
|
||||
// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the
|
||||
// rollout carried once each, and one step for a machine and a tier.
|
||||
func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{
|
||||
"notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}}
|
||||
was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}}
|
||||
|
||||
noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at)
|
||||
noteChanges(&p, "laptop", []string{"notes"}, nil, at)
|
||||
// The send noted again, after what was sent had become what the machine runs: the first before stands.
|
||||
noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute))
|
||||
|
||||
want := []chain.Change{
|
||||
{Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
{Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}},
|
||||
{Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
}
|
||||
if len(p.Changed) != len(want) {
|
||||
t.Fatalf("changed %+v", p.Changed)
|
||||
}
|
||||
for k := range want {
|
||||
if p.Changed[k] != want[k] {
|
||||
t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k])
|
||||
}
|
||||
}
|
||||
if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" {
|
||||
t.Errorf("builds %v", p.Builds)
|
||||
}
|
||||
if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" {
|
||||
t.Errorf("steps %+v", p.Steps)
|
||||
}
|
||||
lines := changedLines(p)
|
||||
if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") ||
|
||||
!strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") {
|
||||
t.Errorf("changed lines %q", lines)
|
||||
}
|
||||
if got := changedLines(inventory.Plan{}); got != nil {
|
||||
t.Errorf("a plan that changed nothing says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
lines := runLines([]chain.Assignment{
|
||||
{Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at},
|
||||
{Node: "anchor", Module: "web", Commit: "c3"},
|
||||
{Node: "anchor", Module: "never-sent"},
|
||||
})
|
||||
if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") ||
|
||||
!strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") {
|
||||
t.Fatalf("lines %q", lines)
|
||||
}
|
||||
if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil {
|
||||
t.Fatal("a module never sent is said to run something")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
|
||||
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
|
||||
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
|
||||
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
|
||||
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(91*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
||||
}
|
||||
rollout := ws[0].ID
|
||||
asking, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ {
|
||||
advancePlans(ctx, open)
|
||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
buildID := asking.Modules["notes"].Build
|
||||
if buildID == "" || len(*asked) != 1 {
|
||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
||||
}
|
||||
|
||||
// What the live path made is in the chain.
|
||||
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
|
||||
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
|
||||
}
|
||||
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
|
||||
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
|
||||
}
|
||||
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, ok := assignmentOf(on, "notes"); !ok {
|
||||
t.Fatalf("the assignment is not read from the chain: %+v", on)
|
||||
}
|
||||
|
||||
// And the catalog of the store holds none of them.
|
||||
have := map[string]bool{}
|
||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||
have[strings.Trim(row[0], `"`)] = true
|
||||
}
|
||||
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
|
||||
if have[table] {
|
||||
t.Errorf("the table %s is in the controller's store", table)
|
||||
}
|
||||
}
|
||||
for _, table := range []string{"build_gate", "assignment_generation"} {
|
||||
if !have[table] {
|
||||
t.Errorf("the table %s, which stays, is gone", table)
|
||||
}
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
||||
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed
|
||||
// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by
|
||||
// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module
|
||||
// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary
|
||||
// send. A pull request's check posts it with its verdict; the release a merge makes follows the same
|
||||
// planner, so the two can be compared.
|
||||
|
||||
// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read.
|
||||
type policyOf func(module string) (inventory.Upgrade, bool)
|
||||
|
||||
// changePlanOf is the change plan of a reach: pure, so it is tested without a store.
|
||||
func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan {
|
||||
p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(),
|
||||
New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
// The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490).
|
||||
holder := ""
|
||||
for _, e := range entries {
|
||||
if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 {
|
||||
holder = e.On[0]
|
||||
}
|
||||
}
|
||||
var granting []string
|
||||
machines := map[string]*link.MachinePlan{}
|
||||
machine := func(name string) *link.MachinePlan {
|
||||
if machines[name] == nil {
|
||||
machines[name] = &link.MachinePlan{Machine: name}
|
||||
}
|
||||
return machines[name]
|
||||
}
|
||||
for _, tier := range r.Plan.Tiers {
|
||||
for _, name := range tier {
|
||||
e, held := byName[name]
|
||||
if !held {
|
||||
continue
|
||||
}
|
||||
u, known := policy(name)
|
||||
waits := known && !u.RollOut
|
||||
for _, on := range e.On {
|
||||
if waits {
|
||||
machine(on).Waits = append(machine(on).Waits, name)
|
||||
} else {
|
||||
machine(on).Receives = append(machine(on).Receives, name)
|
||||
}
|
||||
}
|
||||
if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 &&
|
||||
!slices.Contains(granting, name) {
|
||||
granting = append(granting, name)
|
||||
}
|
||||
switch {
|
||||
case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0:
|
||||
// Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine
|
||||
// until a person takes the step, so merging it is a promise to take it.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+
|
||||
"nothing else is sent to %s either — unless the new build turns out the same as the one running there "+
|
||||
"(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", ")))
|
||||
case name == "nats" || catalogue.ProvidesBus(e.Manifest):
|
||||
p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send")
|
||||
case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData):
|
||||
// Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377).
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+
|
||||
"its move recreates none of its containers and moves none of its data, which the walk judges and then sends",
|
||||
name, orNone(u.From)))
|
||||
case waits && len(e.On) > 0:
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name,
|
||||
orNone(u.From)))
|
||||
}
|
||||
if len(e.Manifest.Provides) > 0 && len(e.On) > 0 {
|
||||
var offers []string
|
||||
for _, o := range e.Manifest.Provides {
|
||||
offers = append(offers, o.Name)
|
||||
}
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it",
|
||||
name, strings.Join(offers, ", ")))
|
||||
}
|
||||
if e.Manifest.Data != nil && len(e.On) > 0 {
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(granting) > 0 {
|
||||
// Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's
|
||||
// user list. Whether it does is known only once the builds are registered, so it is said as a rule.
|
||||
p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+
|
||||
"subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+
|
||||
"machine", grantsStepWord, holder, strings.Join(granting, ", ")))
|
||||
}
|
||||
var names []string
|
||||
for name := range machines {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
p.Machines = append(p.Machines, *machines[name])
|
||||
}
|
||||
p.Summary = summaryOf(p)
|
||||
p.Class = changeClassOf(p, entries)
|
||||
return p
|
||||
}
|
||||
|
||||
// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2).
|
||||
const classNone = "none"
|
||||
|
||||
// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves,
|
||||
// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf
|
||||
// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the
|
||||
// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds
|
||||
// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a
|
||||
// resolver claim its manifest makes.
|
||||
func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string {
|
||||
if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 {
|
||||
return classNone
|
||||
}
|
||||
modules := append(append([]string{}, p.Moved...), p.Dependents...)
|
||||
for _, dir := range p.New {
|
||||
if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" {
|
||||
modules = append(modules, name)
|
||||
}
|
||||
}
|
||||
return classOfModules(modules, entries)
|
||||
}
|
||||
|
||||
// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine
|
||||
// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself.
|
||||
const grantsStepWord = "grants step"
|
||||
|
||||
// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step.
|
||||
const busUpgradeNeeded = "merging this needs a person's bus upgrade"
|
||||
|
||||
// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves.
|
||||
func summaryOf(p link.ChangePlan) string {
|
||||
if len(p.Moved) == 0 && len(p.New) == 0 {
|
||||
return "builds nothing: the change touches no module of the mesh's graph"
|
||||
}
|
||||
var parts []string
|
||||
what := strings.Join(p.Moved, ", ")
|
||||
if len(p.Dependents) > 0 {
|
||||
what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents))
|
||||
}
|
||||
if len(p.New) > 0 {
|
||||
if what != "" {
|
||||
what += ", "
|
||||
}
|
||||
what += "new: " + strings.Join(p.New, ", ")
|
||||
}
|
||||
var to []string
|
||||
for _, m := range p.Machines {
|
||||
if len(m.Receives) > 0 {
|
||||
to = append(to, m.Machine)
|
||||
}
|
||||
}
|
||||
if len(to) > 0 {
|
||||
parts = append(parts, "builds "+what+" → "+strings.Join(to, ", "))
|
||||
} else {
|
||||
parts = append(parts, "builds "+what+", sent nowhere")
|
||||
}
|
||||
bus := "no bus step"
|
||||
for _, s := range p.Steps {
|
||||
switch {
|
||||
case strings.HasPrefix(s, busUpgradeNeeded):
|
||||
bus = busUpgradeNeeded
|
||||
case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step":
|
||||
bus = "a bus step"
|
||||
}
|
||||
}
|
||||
parts = append(parts, bus)
|
||||
waiting := 0
|
||||
for _, m := range p.Machines {
|
||||
waiting += len(m.Waits)
|
||||
}
|
||||
if waiting > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting))
|
||||
}
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in
|
||||
// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a
|
||||
// person, a provider whose consumers follow it.
|
||||
func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) {
|
||||
const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string, on ...string) inventory.Entry {
|
||||
e := fromRepo(name, catalogue_, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = on
|
||||
return e
|
||||
}
|
||||
nats := entry("nats", "modules/nats", "anchor")
|
||||
nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
gitea := entry("gitea", "modules/gitea", "anchor")
|
||||
held := entry("photos", "modules/photos", "anchor", "laptop")
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
tools.On = []string{"anchor", "laptop"}
|
||||
entries := []inventory.Entry{nats, gitea, held, tools}
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(module string) (inventory.Upgrade, bool) {
|
||||
if module == "photos" {
|
||||
return inventory.Upgrade{RollOut: false, From: "a person"}, true
|
||||
}
|
||||
return inventory.Upgrade{RollOut: true}, true
|
||||
}
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
p := plan("modules/gitea/index.ts")
|
||||
if p.Summary != "builds gitea → anchor; no bus step" {
|
||||
t.Errorf("one module's change reads %q", p.Summary)
|
||||
}
|
||||
|
||||
p = plan("modules/nats/Dockerfile", "modules/photos/x.js")
|
||||
if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") ||
|
||||
!strings.Contains(p.Summary, "(+1 dependent(s))") {
|
||||
t.Errorf("the bus and a held module read %q", p.Summary)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, m := range p.Machines {
|
||||
got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",")
|
||||
}
|
||||
// The bus first, what stands on it after: the build plan's order, per machine.
|
||||
if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" {
|
||||
t.Errorf("the deploy plan reads %v", got)
|
||||
}
|
||||
text := strings.Join(p.Steps, "\n")
|
||||
// Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a
|
||||
// person runs the bus's step, and the verb that does.
|
||||
for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor",
|
||||
"mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person",
|
||||
"nats provides mesh-bus"} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the steps do not say %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
p = plan("merge-check.sh")
|
||||
if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" {
|
||||
t.Errorf("a root file's plan reads %+v", p)
|
||||
}
|
||||
}
|
||||
@@ -38,8 +38,10 @@ type checkScope struct {
|
||||
New []string
|
||||
// Manifests are the moved modules' and the new ones' manifests in the change's tree.
|
||||
Manifests []string
|
||||
// Unread the changed files no module's build reads.
|
||||
Unread []string
|
||||
// Width is how many modules a merge would build, in how many tiers; Unread the changed files no
|
||||
// module's build reads.
|
||||
Width, Tiers int
|
||||
Unread []string
|
||||
// Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the
|
||||
// core's, or the change adds a module to it.
|
||||
Mesh bool
|
||||
@@ -60,7 +62,8 @@ func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][]
|
||||
Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs,
|
||||
ModuleDirsSaid: p.ModuleDirsSaid}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, Reach: r}
|
||||
s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread,
|
||||
Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r}
|
||||
for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) {
|
||||
s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile))
|
||||
}
|
||||
|
||||
@@ -93,3 +93,60 @@ func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) {
|
||||
t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request
|
||||
// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no
|
||||
// module in both, and what stands on a touched module is named as its dependents in both.
|
||||
func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) {
|
||||
const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
nats := fromRepo("nats", catalogue, "modules/nats")
|
||||
nats.Source.BuiltFrom = "old"
|
||||
gitea := fromRepo("gitea", catalogue, "modules/gitea")
|
||||
gitea.Source.BuiltFrom = "old"
|
||||
tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools")
|
||||
entries := []inventory.Entry{nats, gitea, tools}
|
||||
// node-tools stands on the bus's image; a code edge, so a plan takes it along.
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}}
|
||||
read := map[string][]inventory.ReadRepository{}
|
||||
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
paths []string
|
||||
moved, dependents string
|
||||
width int
|
||||
unread string
|
||||
}{
|
||||
{"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"},
|
||||
{"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""},
|
||||
{"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"},
|
||||
} {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
s := pullScope(p, entries, read, edges)
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths}
|
||||
r := reachOfMerge(m, entries, read, edges)
|
||||
if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") {
|
||||
t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved)
|
||||
}
|
||||
if got := strings.Join(s.Dependents, ","); got != c.dependents {
|
||||
t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents)
|
||||
}
|
||||
if s.Width != c.width || s.Width != len(r.Plan.Modules) {
|
||||
t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width)
|
||||
}
|
||||
if got := strings.Join(s.Unread, ","); got != c.unread {
|
||||
t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread)
|
||||
}
|
||||
}
|
||||
|
||||
// A repository whose build another module's recipe packages: that module is reached through the
|
||||
// planner's `read`, and what stands on it after it.
|
||||
read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}}
|
||||
s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head",
|
||||
Paths: []string{"src/index.ts"}}, entries, read, edges)
|
||||
if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 {
|
||||
t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
||||
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
||||
|
||||
@@ -245,8 +245,41 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
|
||||
if out.Sequence, err = inv.Sequence(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
// Every node-engine of release 6 reads a module per resource and a build per module.
|
||||
out.ReadsModules = true
|
||||
if out.Epoch, err = inv.SentEpoch(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.Generation, err = inv.SentGeneration(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.ReadsModules, err = inv.ReadsModules(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil {
|
||||
return out, err
|
||||
}
|
||||
runs, err := inv.AssignmentRuns(ctx, n.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if len(runs) > 0 {
|
||||
out.Runs = make(map[string]compose.Run, len(runs))
|
||||
for module, ref := range runs {
|
||||
out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build}
|
||||
}
|
||||
}
|
||||
// A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its
|
||||
// manifest and the commit, applied to this node's own resolution only.
|
||||
// What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it).
|
||||
kept, err := recordedKept(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
*refused = err
|
||||
} else if len(kept) > 0 {
|
||||
out.KeptBuilds = kept
|
||||
if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil {
|
||||
*refused = err
|
||||
out.KeptBuilds, out.Kept = nil, nil
|
||||
}
|
||||
}
|
||||
stated, has, err := inv.HealthOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
@@ -258,6 +291,18 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keptManifests is the manifest of each module kept at the build a machine runs: keepRecorded's reads, and its refusals in its
|
||||
// words, without putting any of it into a shelf (that is the node's own Kept, applied by the composition).
|
||||
func keptManifests(ctx context.Context, open *stores, node string, kept map[string]string) (map[string]catalogue.Manifest, error) {
|
||||
// applyKept puts each manifest into the shelf it is given and refuses a build that cannot be read; the shelf here is a
|
||||
// scratch one, whose entries are the kept manifests.
|
||||
scratch := map[string]catalogue.Manifest{}
|
||||
if err := applyKept(ctx, open, node, kept, scratch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scratch, nil
|
||||
}
|
||||
|
||||
func readTunnels(ctx context.Context, inv *inventory.Inventory) (map[string]compose.FoundTunnel, error) {
|
||||
found, err := inv.Tunnels(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -6,9 +6,11 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1382: the adapter's Input names, for every node, what the inventory answers for it, each compared with the read named
|
||||
@@ -130,6 +132,69 @@ func nodePorts(in compose.Input, name string) map[string]map[int]int {
|
||||
return n.Ports
|
||||
}
|
||||
|
||||
// 1382/AC2: a node keeping a recorded module at the build it runs has that module in Kept and KeptBuilds, and the Input's
|
||||
// Catalogue is the current shelf, not the kept manifest (R4a review round 1, finding 1).
|
||||
func TestANodeKeepingARecordedBuildHasItInKeptAndTheCatalogueStaysCurrent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
image := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64)
|
||||
for _, b := range []link.BuildResult{
|
||||
aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, map[string][2]string{"server": {image, ""}}),
|
||||
} {
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"postgres": "c1111111"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute),
|
||||
map[string][2]string{"server": {image, "runtime"}})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A send that is not a person's push keeps the build the machine runs.
|
||||
in, err := readComposeInput(keepingRecorded(ctx), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
anchor, _ := nodeOf(in.Input, "anchor")
|
||||
if anchor.KeptBuilds["postgres"] != "c1111111" {
|
||||
t.Fatalf("anchor keeps %v, want postgres at c1111111", anchor.KeptBuilds)
|
||||
}
|
||||
kept, found := anchor.Kept["postgres"]
|
||||
if !found {
|
||||
t.Fatalf("the kept manifest of postgres is not in anchor's Kept: %v", anchor.Kept)
|
||||
}
|
||||
current, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(in.Catalogue["postgres"], current["postgres"]) {
|
||||
t.Error("the Input's Catalogue is not the current shelf")
|
||||
}
|
||||
if reflect.DeepEqual(kept, current["postgres"]) {
|
||||
t.Error("the kept manifest is the current one: nothing is kept")
|
||||
}
|
||||
if laptop, _ := nodeOf(in.Input, "laptop"); len(laptop.Kept) != 0 || len(laptop.KeptBuilds) != 0 {
|
||||
t.Errorf("laptop keeps what only anchor runs: %v", laptop.KeptBuilds)
|
||||
}
|
||||
|
||||
// A person's push keeps nothing.
|
||||
pushed, err := readComposeInput(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a, _ := nodeOf(pushed.Input, "anchor"); len(a.Kept) != 0 || len(a.KeptBuilds) != 0 {
|
||||
t.Errorf("a person's push keeps %v", a.KeptBuilds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1382/AC3: the seats on record, the tunnels, the adopted tunnel, its carried peers and the nodes joining equal the
|
||||
// inventory's answers; the range and the suffix are read from the environment once, here.
|
||||
func TestTheAdaptersInputCarriesTheNetworkAsTheInventoryHoldsIt(t *testing.T) {
|
||||
|
||||
@@ -39,10 +39,24 @@ func withInputCache(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
|
||||
}
|
||||
|
||||
// freshInput is ctx for one composition whose Input is read afresh and kept only until the composition makes something: a
|
||||
// send that is composed after another one made and recorded things (a push's machines, one after the other).
|
||||
func freshInput(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
|
||||
}
|
||||
|
||||
// keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs
|
||||
// (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send.
|
||||
func keptSignature(ctx context.Context) string {
|
||||
return ""
|
||||
skip, on := keptExcept(ctx)
|
||||
var names []string
|
||||
for m := range skip {
|
||||
names = append(names, m)
|
||||
}
|
||||
sort.Strings(names)
|
||||
every, _ := ctx.Value(keepEveryBuildKey{}).(string)
|
||||
settings, _ := ctx.Value(keepForSettingsKey{}).(string)
|
||||
return fmt.Sprintf("kept=%v;except=%v;every=%s;settings=%s", on, names, every, settings)
|
||||
}
|
||||
|
||||
// inputFor is the Input for ctx's operation.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -85,3 +86,26 @@ func TestADeclarationThatStandsInForAValueIsStillForeseen(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1386: an Input read for a person's push is never the one read for a send that keeps recorded builds, and the cache is
|
||||
// emptied by what a send made.
|
||||
func TestAnInputIsCachedPerWhatItKeepsAndForgottenAfterASend(t *testing.T) {
|
||||
ctx := withInputCache(context.Background())
|
||||
plain := keptSignature(ctx)
|
||||
kept := keptSignature(keepingRecorded(ctx))
|
||||
bus := keptSignature(keepingRecorded(ctx, "nats"))
|
||||
every := keptSignature(keepingEveryBuild(ctx, "anchor"))
|
||||
for name, sig := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
|
||||
for other, o := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} {
|
||||
if name != other && sig == o {
|
||||
t.Errorf("%s and %s read the same Input", name, other)
|
||||
}
|
||||
}
|
||||
}
|
||||
cache := ctx.Value(inputCacheKey{}).(*inputCache)
|
||||
cache.by[plain] = &meshInput{}
|
||||
forgetInput(ctx)
|
||||
if len(cache.by) != 0 {
|
||||
t.Fatal("what a send made left the Input it read in the cache")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,143 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// A mesh that composes (novox/hq ADR 0539): the real store (theCatalogue: sudo from the catalogue, mesh-host from the
|
||||
// node-engine's repository, the private network on anchor and laptop), the controller's signing key in its own
|
||||
// identity store, and a real bus server this process composes onto, its lease taken by the act as a command's is. Every
|
||||
// subject published on the bus is heard, so a test can say nothing went to a node's declaration subject.
|
||||
type composingMesh struct {
|
||||
open *stores
|
||||
js jetstream.JetStream
|
||||
conn *nats.Conn
|
||||
mu sync.Mutex
|
||||
heard []string
|
||||
}
|
||||
|
||||
func aComposingMesh(t *testing.T) *composingMesh {
|
||||
t.Helper()
|
||||
open := theCatalogue(t)
|
||||
ident := identity.ForTest(t)
|
||||
if _, err := ident.Establish(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := testbus.Start(t)
|
||||
t.Setenv(broker.NATSVar, s.ClientURL())
|
||||
was := theLease
|
||||
theLease = &actor{}
|
||||
t.Cleanup(func() { theLease.release(); theLease = was })
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := &composingMesh{open: open, js: js, conn: conn}
|
||||
sub, err := conn.Subscribe(">", func(msg *nats.Msg) {
|
||||
m.mu.Lock()
|
||||
m.heard = append(m.heard, msg.Subject)
|
||||
m.mu.Unlock()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
legacy, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller's seat events, kept as the mesh's events stream keeps them.
|
||||
if _, err := legacy.AddStream(&nats.StreamConfig{Name: "CONTROLLER_EVENTS", Subjects: []string{"mesh.seat.mesh-controller.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasEvents := checkEvents
|
||||
checkEvents = link.OverNATS{Conn: conn, JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = wasEvents })
|
||||
withTrunkList(t, true)
|
||||
return m
|
||||
}
|
||||
|
||||
// trunkKeys are a done key and the indexer's trunk value naming it.
|
||||
type trunkKeys struct {
|
||||
keyBuckets
|
||||
trunk sdkdelivery.Module
|
||||
}
|
||||
|
||||
func (k trunkKeys) ModuleAt(context.Context, string, string) (sdkdelivery.Module, error) {
|
||||
return k.trunk, nil
|
||||
}
|
||||
|
||||
// 1945/AC14: the declarations' stream stays while one node's have names another node-engine than the one registered,
|
||||
// and goes, said once, when every node's names it (ADR 0539 clause 11).
|
||||
func TestTheDeclarationsStreamGoesOnceEveryNodeRunsTheNewEngine(t *testing.T) {
|
||||
m := aComposingMesh(t)
|
||||
ctx := t.Context()
|
||||
legacy, err := m.conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := legacy.AddStream(&nats.StreamConfig{Name: broker.DeclarationsStream, Subjects: []string{"mesh.node.*.declare"},
|
||||
MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The new node-engine registered at its fingerprint, from its done key.
|
||||
n6 := fpOf('6')
|
||||
keys, trunk := aKeyFor("mesh-host", "6", n6, "novox/mesh-host", "", "f00d0123456789ab")
|
||||
if _, err := liveDoneKeySteps(m.open, trunkKeys{keyBuckets: keys, trunk: trunk}).take(ctx, "mesh-host"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: sdkdelivery.HaveBucket, History: sdkdelivery.HaveHistory}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
haves, err := m.js.KeyValue(ctx, sdkdelivery.HaveBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have := func(node, fp string) {
|
||||
key, _ := sdkdelivery.HaveKey(node, "mesh-host")
|
||||
raw, _ := sdkdelivery.EncodeHave(sdkdelivery.Have{FP: fp, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK})
|
||||
if _, err := haves.Put(ctx, key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
have("anchor", n6)
|
||||
have("laptop", fpOf('5'))
|
||||
if done, err := retireDeclarations(ctx, m.open); err != nil || done {
|
||||
t.Fatalf("with laptop on the old node-engine: done %v (%v)", done, err)
|
||||
}
|
||||
if _, err := m.js.Stream(ctx, broker.DeclarationsStream); err != nil {
|
||||
t.Fatalf("the stream went while laptop runs the old node-engine: %v", err)
|
||||
}
|
||||
have("laptop", n6)
|
||||
if done, err := retireDeclarations(ctx, m.open); err != nil || !done {
|
||||
t.Fatalf("with every node on the new node-engine: done %v (%v)", done, err)
|
||||
}
|
||||
if _, err := m.js.Stream(ctx, broker.DeclarationsStream); !errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
t.Errorf("the stream is still there: %v", err)
|
||||
}
|
||||
if done, err := retireDeclarations(ctx, m.open); err != nil || !done {
|
||||
t.Errorf("a second look: done %v (%v)", done, err)
|
||||
}
|
||||
if !declarationsRetiredNow(ctx, m.open.inventory) {
|
||||
t.Errorf("a raise after the retirement would assert the stream again")
|
||||
}
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The judgement of a move (novox/hq ADR 0539 clauses 2 and 14, by ADR 0377 §3's tests): beside each composition's digest
|
||||
// the controller writes whether moving the node from the composition it runs, as its have names it, to this one
|
||||
// recreates a container, moves the module's data, or neither, or that it cannot be judged. The assigner holds a move of
|
||||
// a module of policy record from it until a person allows it.
|
||||
//
|
||||
// ADR 0377 §3 over two compositions of one module: **recreates** when a container resource is changed, added or
|
||||
// removed; **moves data** when, with no container changed, a directory or file resource is changed, added or removed
|
||||
// that a container of the module mounts or that holds or lies within one of its directories (where a module keeps its
|
||||
// data), when its `data` declaration changes, or, for a module that declares data, when one of its long-running
|
||||
// processes changes; **neither** otherwise, and for a module the node has never run. **Cannot judge** when the
|
||||
// composition the node runs is gone or cannot be read. A held composition never becomes what is judged against: with A
|
||||
// running, B held and C composed, C is judged against A, the one the have names.
|
||||
|
||||
// judgeMove is the value written beside a composition whose digest is digest: its judgement against what the node's have
|
||||
// names for the module, and the module's policy.
|
||||
func judgeMove(ctx context.Context, at composedPlace, key string, to sdkdelivery.Composed, digest string) (sdkdelivery.ComposedDigest, error) {
|
||||
out := sdkdelivery.ComposedDigest{Digest: digest}
|
||||
if at.policy != nil && to.Module != "" {
|
||||
out.Policy, out.PolicyFrom, out.PolicyWhy = at.policy(to.Module)
|
||||
}
|
||||
if to.Module == "" {
|
||||
// The node's own composition: no module's containers or data are judged; its want is written as any.
|
||||
out.Judgement = sdkdelivery.JudgedNeither
|
||||
return out, nil
|
||||
}
|
||||
have, found, err := at.ran(ctx, to.Module)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
// A node whose have names no composition yet (the switch: it runs the declaration it last applied) is judged
|
||||
// against that module's slice of that declaration (novox/hq ADR 0540 clause 9): equal is unchanged, a want that
|
||||
// changes nothing on the node; otherwise by ADR 0377 §3's tests, as any move.
|
||||
if found && have.Composed == "" && have.FP != "" && have.Phase != sdkdelivery.PhaseRemoved && at.lastApplied != nil {
|
||||
slice, declared, applied, err := at.lastApplied(ctx, to.Module)
|
||||
if err != nil {
|
||||
// Unread is not judged: a person's yes holds it, and the words say why.
|
||||
out.Judgement = sdkdelivery.JudgedCannot
|
||||
out.Words = fmt.Sprintf("%s's have on %s names no composition, and the declaration it last applied cannot be read: %v",
|
||||
to.Module, at.node, err)
|
||||
return out, nil
|
||||
}
|
||||
if applied {
|
||||
from := sdkdelivery.Composed{Node: at.node, Module: to.Module, FP: have.FP, Data: to.Data, Resources: slice}
|
||||
if sameResources(from, to) {
|
||||
out.Judgement, out.Words = sdkdelivery.JudgedUnchanged, "equal to the declaration "+at.node+" last applied"
|
||||
return out, nil
|
||||
}
|
||||
// Against names the declaration, by its digest: the composition the node runs is that declaration's slice.
|
||||
out.Against = declared
|
||||
if onlyTheUserList(from, to) {
|
||||
// Only the bus's user list, as against a composition (ADR 0539 clause 7): no yes waits on it at the switch.
|
||||
out.Judgement, out.UserListOnly = sdkdelivery.JudgedNeither, true
|
||||
out.Words = "only the bus's user list changes, against the declaration " + at.node + " last applied"
|
||||
return out, nil
|
||||
}
|
||||
j, words := judged(from, to)
|
||||
out.Judgement, out.Words = j, words+", against the declaration "+at.node+" last applied"
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
if !found || have.Phase == sdkdelivery.PhaseRemoved || (have.Composed == "" && have.FP == "") {
|
||||
out.Judgement, out.Words = sdkdelivery.JudgedNeither, "the node runs no composition of "+to.Module
|
||||
return out, nil
|
||||
}
|
||||
if have.Composed == "" {
|
||||
out.Judgement = sdkdelivery.JudgedCannot
|
||||
out.Words = fmt.Sprintf("%s's have on %s names no composition, only the fingerprint %s", to.Module, at.node, have.FP)
|
||||
return out, nil
|
||||
}
|
||||
out.Against = have.Composed
|
||||
if have.Composed == digest {
|
||||
out.Judgement, out.Words = sdkdelivery.JudgedNeither, "the node runs this composition"
|
||||
return out, nil
|
||||
}
|
||||
from, found, err := composedByDigest(ctx, at, to.Module, have.FP, have.Composed)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !found {
|
||||
out.Judgement = sdkdelivery.JudgedCannot
|
||||
out.Words = fmt.Sprintf("the composition %s runs on %s (%s) is gone", to.Module, at.node, short(strings.TrimPrefix(have.Composed, "sha256:")))
|
||||
return out, nil
|
||||
}
|
||||
if onlyTheUserList(from, to) {
|
||||
// Only the bus's user list (ADR 0539 clause 7): a file change and a reload of the bus, no move of it, written
|
||||
// into its want at once.
|
||||
out.Judgement, out.UserListOnly, out.Words = sdkdelivery.JudgedNeither, true, "only the bus's user list changes"
|
||||
return out, nil
|
||||
}
|
||||
out.Judgement, out.Words = judged(from, to)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busUsersSuffix ends the id of the bus module's user list as the composer writes it: `<module>.bus-users`.
|
||||
const busUsersSuffix = ".bus-users"
|
||||
|
||||
// onlyTheUserList says two compositions of a module differ, and only in its file of the bus's user list.
|
||||
func onlyTheUserList(from, to sdkdelivery.Composed) bool {
|
||||
if from.Data != to.Data {
|
||||
return false
|
||||
}
|
||||
before, after := resourcesByID(from), resourcesByID(to)
|
||||
if len(before) != len(after) {
|
||||
return false
|
||||
}
|
||||
changed := false
|
||||
for id, now := range after {
|
||||
was, ok := before[id]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if bytes.Equal(was.raw, now.raw) {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(id, busUsersSuffix) || now.kind != string(sdkdecl.TypeFile) {
|
||||
return false
|
||||
}
|
||||
changed = true
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// composedByDigest finds, in the node's bucket, the composition of module at fingerprint fp whose digest is digest: the
|
||||
// key's current value or one of the revisions it keeps.
|
||||
func composedByDigest(ctx context.Context, at composedPlace, module, fp, digest string) (sdkdelivery.Composed, bool, error) {
|
||||
if fp == "" {
|
||||
return sdkdelivery.Composed{}, false, nil
|
||||
}
|
||||
key, err := sdkdelivery.ComposedKey(at.node, module, fp)
|
||||
if err != nil {
|
||||
return sdkdelivery.Composed{}, false, nil
|
||||
}
|
||||
values, err := historyOf(ctx, at.bucket, key)
|
||||
if err != nil {
|
||||
return sdkdelivery.Composed{}, false, err
|
||||
}
|
||||
for _, v := range values {
|
||||
c, err := sdkdelivery.DecodeComposed(v)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if d, err := sdkdelivery.ComposedDigestOf(c); err == nil && d == digest {
|
||||
return c, true, nil
|
||||
}
|
||||
}
|
||||
return sdkdelivery.Composed{}, false, nil
|
||||
}
|
||||
|
||||
// historyOf is every value a key keeps, the newest first; none when there is no such key.
|
||||
func historyOf(ctx context.Context, kv composedBucketsKV, key string) ([][]byte, error) {
|
||||
if h, ok := kv.(interface {
|
||||
History(ctx context.Context, key string, opts ...jetstream.WatchOpt) ([]jetstream.KeyValueEntry, error)
|
||||
}); ok {
|
||||
entries, err := h.History(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([][]byte, 0, len(entries))
|
||||
for i := len(entries) - 1; i >= 0; i-- {
|
||||
if entries[i].Operation() == jetstream.KeyValuePut {
|
||||
out = append(out, entries[i].Value())
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return [][]byte{e.Value()}, nil
|
||||
}
|
||||
|
||||
// judged applies ADR 0377 §3 to two compositions of one module.
|
||||
func judged(from, to sdkdelivery.Composed) (sdkdelivery.Judgement, string) {
|
||||
before, after := resourcesByID(from), resourcesByID(to)
|
||||
if ids := changedOf(before, after, func(kind string) bool { return kind == string(sdkdecl.TypeContainer) }); len(ids) > 0 {
|
||||
return sdkdelivery.JudgedRecreates, "it recreates " + strings.Join(ids, ", ")
|
||||
}
|
||||
if from.Data != to.Data {
|
||||
return sdkdelivery.JudgedMovesData, "its data declaration changes"
|
||||
}
|
||||
places := dataPlaces(before, after)
|
||||
held := func(kind string) bool {
|
||||
return kind == string(sdkdecl.TypeDirectory) || kind == string(sdkdecl.TypeFile)
|
||||
}
|
||||
var moved []string
|
||||
for _, id := range changedOf(before, after, held) {
|
||||
r := before[id]
|
||||
if r == nil {
|
||||
r = after[id]
|
||||
}
|
||||
if withinPlaces(r.path, places) {
|
||||
moved = append(moved, id)
|
||||
}
|
||||
}
|
||||
if to.Data || from.Data {
|
||||
moved = append(moved, changedOf(before, after, func(kind string) bool { return kind == string(sdkdecl.TypeProcess) })...)
|
||||
}
|
||||
if len(moved) > 0 {
|
||||
sort.Strings(moved)
|
||||
return sdkdelivery.JudgedMovesData, "it changes where its data is kept: " + strings.Join(moved, ", ")
|
||||
}
|
||||
return sdkdelivery.JudgedNeither, "it recreates no container and moves no data"
|
||||
}
|
||||
|
||||
// judgedResource is a resource as the judgement reads it: its kind, the path it acts on, what its containers mount,
|
||||
// and its canonical bytes.
|
||||
type judgedResource struct {
|
||||
kind string
|
||||
path string
|
||||
mounts []string
|
||||
raw []byte
|
||||
}
|
||||
|
||||
func resourcesByID(c sdkdelivery.Composed) map[string]*judgedResource {
|
||||
out := map[string]*judgedResource{}
|
||||
for _, raw := range c.Resources {
|
||||
var head struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
// A container's volumes, `<path on the machine>:<path inside>[:<options>]`.
|
||||
Volumes []string `json:"volumes"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &head); err != nil {
|
||||
continue
|
||||
}
|
||||
r := &judgedResource{kind: head.Type, path: head.Path, raw: compactComposed(raw)}
|
||||
for _, v := range head.Volumes {
|
||||
source, _, _ := strings.Cut(v, ":")
|
||||
r.mounts = append(r.mounts, source)
|
||||
}
|
||||
out[head.ID] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// compactComposed is a resource's bytes canonical: its keys sorted, its spacing dropped, its numbers as written, so
|
||||
// two compositions that say the same compare equal however each was written.
|
||||
func compactComposed(raw []byte) []byte {
|
||||
var v any
|
||||
d := json.NewDecoder(bytes.NewReader(raw))
|
||||
d.UseNumber()
|
||||
if err := d.Decode(&v); err != nil {
|
||||
return raw
|
||||
}
|
||||
out, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changedOf are the ids of the resources of the kinds counted that are changed, added or removed, sorted.
|
||||
func changedOf(before, after map[string]*judgedResource, counted func(kind string) bool) []string {
|
||||
var out []string
|
||||
for id, now := range after {
|
||||
if !counted(now.kind) {
|
||||
continue
|
||||
}
|
||||
if was, ok := before[id]; !ok || !bytes.Equal(was.raw, now.raw) {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
for id, was := range before {
|
||||
if _, ok := after[id]; !ok && counted(was.kind) {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// dataPlaces are the paths where the module keeps what a move must not lose: what its containers mount, and its
|
||||
// directories, before and after.
|
||||
func dataPlaces(before, after map[string]*judgedResource) []string {
|
||||
var out []string
|
||||
for _, rs := range []map[string]*judgedResource{before, after} {
|
||||
for _, r := range rs {
|
||||
if r.kind == string(sdkdecl.TypeContainer) {
|
||||
out = append(out, r.mounts...)
|
||||
}
|
||||
if r.kind == string(sdkdecl.TypeDirectory) && r.path != "" {
|
||||
out = append(out, r.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withinPlaces says p is one of places, or lies within one, or holds one.
|
||||
func withinPlaces(p string, places []string) bool {
|
||||
if p == "" {
|
||||
return false
|
||||
}
|
||||
p = path.Clean(p)
|
||||
for _, place := range places {
|
||||
if place == "" || !strings.HasPrefix(place, "/") {
|
||||
continue
|
||||
}
|
||||
place = path.Clean(place)
|
||||
if p == place || strings.HasPrefix(p, place+"/") || strings.HasPrefix(place, p+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sameResources says two compositions of a module hold the same resources in the same order, each compared canonical.
|
||||
func sameResources(a, b sdkdelivery.Composed) bool {
|
||||
if len(a.Resources) != len(b.Resources) {
|
||||
return false
|
||||
}
|
||||
for i := range a.Resources {
|
||||
if !bytes.Equal(compactComposed(a.Resources[i]), compactComposed(b.Resources[i])) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// mailOn is a composition of mesh-mail on ace at fingerprint c, from the given resources, each saying its module.
|
||||
func mailOn(t *testing.T, c byte, data bool, resources ...string) sdkdelivery.Composed {
|
||||
t.Helper()
|
||||
out := sdkdelivery.Composed{Node: "ace", Module: "mesh-mail", FP: sdkdelivery.SourcePrefix2 + strings.Repeat(string(c), 64), Data: data}
|
||||
for _, r := range resources {
|
||||
out.Resources = append(out.Resources, json.RawMessage(r))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const (
|
||||
mailContainer = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:1111111111111111111111111111111111111111111111111111111111111111","volumes":["/var/lib/mesh-mail/spool:/spool"]}`
|
||||
mailContainer2 = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:2222222222222222222222222222222222222222222222222222222222222222","volumes":["/var/lib/mesh-mail/spool:/spool"]}`
|
||||
mailSpool = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/var/lib/mesh-mail/spool"}`
|
||||
mailSpoolMoved = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/srv/mesh-mail/spool"}`
|
||||
mailTools = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"a"}`
|
||||
mailTools2 = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"b"}`
|
||||
mailConfig = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"a"}`
|
||||
mailConfig2 = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"b"}`
|
||||
)
|
||||
|
||||
// 1945/AC12: captured pairs of compositions are judged by ADR 0377 §3's tests: a changed container recreates; with no
|
||||
// container changed, a directory moved, a file within the data a container mounts changed, or the data declaration
|
||||
// changed moves data; a tools file nothing mounts changed is neither.
|
||||
func TestAMoveIsJudgedByWhatItRecreatesOrMoves(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
from, to sdkdelivery.Composed
|
||||
want sdkdelivery.Judgement
|
||||
}{
|
||||
{"a new image", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates},
|
||||
{"a container gone", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailSpool), sdkdelivery.JudgedRecreates},
|
||||
{"its spool moved", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpoolMoved), sdkdelivery.JudgedMovesData},
|
||||
{"a file in its spool", mailOn(t, 'a', true, mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData},
|
||||
{"its data declaration", mailOn(t, 'a', false, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpool), sdkdelivery.JudgedMovesData},
|
||||
{"its tools only", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got, words := judged(c.from, c.to); got != c.want || words == "" {
|
||||
t.Errorf("%s: judged %s (%q), want %s", c.name, got, words, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC12: a move is judged against the composition the node's have names: one the node never ran is neither; one
|
||||
// whose composition is gone cannot be judged and says why; and with A running, B held and C composed, C is judged
|
||||
// against A, and judged again when the have changes.
|
||||
func TestAMoveIsJudgedAgainstWhatTheNodeRuns(t *testing.T) {
|
||||
at, _, _, haves := aPlace(t, "ace")
|
||||
ctx := context.Background()
|
||||
a := mailOn(t, 'a', true, mailContainer, mailSpool, mailTools)
|
||||
b := mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools)
|
||||
c := mailOn(t, 'c', true, mailContainer, mailSpool, mailTools2)
|
||||
write := func(comp sdkdelivery.Composed) (string, string) {
|
||||
key, _ := sdkdelivery.ComposedKey("ace", comp.Module, comp.FP)
|
||||
if _, err := writeComposed(ctx, at, key, comp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, _ := sdkdelivery.ComposedDigestOf(comp)
|
||||
return key, d
|
||||
}
|
||||
judgedNow := func(key string) sdkdelivery.ComposedDigest {
|
||||
e, err := at.digests.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := sdkdelivery.DecodeComposedDigest(e.Value())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
keyA, digestA := write(a)
|
||||
if v := judgedNow(keyA); v.Judgement != sdkdelivery.JudgedNeither || v.Against != "" {
|
||||
t.Errorf("a module the node never ran: %+v", v)
|
||||
}
|
||||
if v := judgedNow(keyA); v.Policy != "roll" || v.PolicyFrom != "default" {
|
||||
t.Errorf("the policy beside the judgement: %+v", v)
|
||||
}
|
||||
haves["mesh-mail"] = sdkdelivery.Have{FP: a.FP, Composed: digestA, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
keyB, _ := write(b)
|
||||
if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedRecreates || v.Against != digestA {
|
||||
t.Errorf("B against A: %+v", v)
|
||||
}
|
||||
// B is held (the assigner writes no want for it); the node still runs A.
|
||||
keyC, digestC := write(c)
|
||||
if v := judgedNow(keyC); v.Judgement != sdkdelivery.JudgedNeither || v.Against != digestA {
|
||||
t.Errorf("C against A, B held: %+v", v)
|
||||
}
|
||||
// The have changes to C: every judgement of the module is written again, against what runs now (the have watch).
|
||||
haves["mesh-mail"] = sdkdelivery.Have{FP: c.FP, Composed: digestC, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
if n, err := rejudge(ctx, at, "mesh-mail"); err != nil || n == 0 {
|
||||
t.Fatalf("rejudged %d (%v)", n, err)
|
||||
}
|
||||
if v := judgedNow(keyC); v.Against != digestC || v.Judgement != sdkdelivery.JudgedNeither {
|
||||
t.Errorf("C once the node runs it: %+v", v)
|
||||
}
|
||||
if v := judgedNow(keyB); v.Against != digestC || v.Judgement != sdkdelivery.JudgedRecreates {
|
||||
t.Errorf("B once the node runs C: %+v", v)
|
||||
}
|
||||
// A person sets the module's policy: written beside every composition of it.
|
||||
at.policy = func(string) (string, string, string) { return "record", "person", "its spool" }
|
||||
if _, err := rejudge(ctx, at, "mesh-mail"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := judgedNow(keyB); v.Policy != "record" || v.PolicyFrom != "person" || v.PolicyWhy != "its spool" {
|
||||
t.Errorf("the person's policy beside B: %+v", v)
|
||||
}
|
||||
// A have naming a composition the buckets no longer hold.
|
||||
haves["mesh-mail"] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("9", 64), Composed: "sha256:" + strings.Repeat("9", 64),
|
||||
Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
write(b)
|
||||
if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedCannot || !strings.Contains(v.Words, "gone") {
|
||||
t.Errorf("against a composition that is gone: %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
busServer = `{"id":"nats.server","type":"container","module":"nats","name":"nats","image":"nats@sha256:4444444444444444444444444444444444444444444444444444444444444444"}`
|
||||
busServer2 = `{"id":"nats.server","type":"container","module":"nats","name":"nats","image":"nats@sha256:5555555555555555555555555555555555555555555555555555555555555555"}`
|
||||
busUsers = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users.conf","content":"users: [a]"}`
|
||||
busUsers2 = `{"id":"nats.bus-users","type":"file","module":"nats","path":"/etc/nats/users.conf","content":"users: [a, b]"}`
|
||||
)
|
||||
|
||||
// busOn is a composition of the bus on ace from the given resources.
|
||||
func busOn(t *testing.T, resources ...string) sdkdelivery.Composed {
|
||||
t.Helper()
|
||||
out := sdkdelivery.Composed{Node: "ace", Module: "nats", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64)}
|
||||
for _, r := range resources {
|
||||
out.Resources = append(out.Resources, json.RawMessage(r))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ADR 0540 clause 9: a have that names no composition, only a fingerprint (the switch: the node runs the declaration it
|
||||
// last applied), is judged against the module's slice of that declaration, read from the kept stream of declarations:
|
||||
// an equal slice is unchanged; a slice differing only in the bus's user list is that (round 1's A2, so no yes waits on it at the switch); a changed setting is judged as any move; no declaration to read cannot be judged.
|
||||
func TestAHaveWithNoCompositionIsJudgedAgainstTheLastDeclaration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
other := `{"id":"mesh-calendar.server","type":"container","module":"mesh-calendar","name":"calendar","image":"cal@sha256:3333333333333333333333333333333333333333333333333333333333333333"}`
|
||||
unsaid := `{"id":"mesh-mail.tools","type":"file","path":"/usr/lib/mesh-mail/tools.json","content":"a"}`
|
||||
declared := func(resources ...string) []byte {
|
||||
body, err := json.Marshal(map[string]any{"node": "ace", "resources": json.RawMessage("[" + strings.Join(resources, ",") + "]")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signed, err := json.Marshal(link.Signed{Declaration: body, Signature: []byte("s")}) // as link.Envelope sent it
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return signed
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
last []byte // the node's last declaration on the stream; nil when there is none
|
||||
to sdkdelivery.Composed
|
||||
want sdkdelivery.Judgement
|
||||
inWords string
|
||||
userList bool
|
||||
}{
|
||||
{"an equal slice", declared(mailContainer, other, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedUnchanged, "last applied", false},
|
||||
{"a declared resource saying no module is not equal", declared(mailContainer, other, mailSpool, unsaid), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedNeither, "last applied", false},
|
||||
{"a changed image", declared(mailContainer, other, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates, "last applied", false},
|
||||
{"a changed setting in its spool", declared(mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData, "last applied", false},
|
||||
{"a changed tools file", declared(mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither, "last applied", false},
|
||||
{"only the bus's user list", declared(busServer, busUsers, other), busOn(t, busServer, busUsers2), sdkdelivery.JudgedNeither, "only the bus's user list", true},
|
||||
{"the bus's user list and its container", declared(busServer, busUsers), busOn(t, busServer2, busUsers2), sdkdelivery.JudgedRecreates, "last applied", false},
|
||||
{"no declaration kept", nil, mailOn(t, 'b', true, mailContainer, mailSpool, mailTools), sdkdelivery.JudgedCannot, "names no composition", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
at, _, _, haves := aPlace(t, "ace")
|
||||
last := c.last
|
||||
at.lastApplied = func(_ context.Context, module string) ([]json.RawMessage, string, bool, error) {
|
||||
if last == nil {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
return sliceOfDeclaration(last, module)
|
||||
}
|
||||
haves[c.to.Module] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
key, _ := sdkdelivery.ComposedKey("ace", c.to.Module, c.to.FP)
|
||||
digest, err := sdkdelivery.ComposedDigestOf(c.to)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
got, err := judgeMove(ctx, at, key, c.to, digest)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.name, err)
|
||||
}
|
||||
if got.UserListOnly != c.userList {
|
||||
t.Errorf("%s: user list only %v, want %v", c.name, got.UserListOnly, c.userList)
|
||||
}
|
||||
if got.Judgement != c.want || !strings.Contains(got.Words, c.inWords) {
|
||||
t.Errorf("%s: judged %s (%q), want %s", c.name, got.Judgement, got.Words, c.want)
|
||||
}
|
||||
if _, err := sdkdelivery.EncodeComposedDigest(got); err != nil {
|
||||
t.Errorf("%s: the judgement written is refused: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,264 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A new fingerprint of a registered module, from its done key to its compositions (novox/hq ADR 0539 clauses 5 and 7,
|
||||
// ADR 0501 §2). With no delivery plan left to take a module's done key in, a write of the builder's or the indexer's
|
||||
// keys does it: the controller takes in the done key of the fingerprint the indexer's trunk value names (registers the
|
||||
// module at it, as a plan's take-in did), composes every node the module is assigned to (which makes what a module's
|
||||
// new manifest needs, a credential among it, as a send made it), composes the node holding the bus (its user list now
|
||||
// carries the module's permissions at the new fingerprint: a new revision of the bus module's composition alone,
|
||||
// written into its want at once by the assigner), and writes `grants.<module>.<fp>`. The assigner's sixth and seventh
|
||||
// rules then let a want name the fingerprint. **No timer**: woken by the keys, and a starting controller looks at every
|
||||
// module once (B5's start pass).
|
||||
|
||||
// doneKeySteps are what taking a module's done key in does, each a step a test may stand in for.
|
||||
type doneKeySteps struct {
|
||||
// held says the catalogue holds the module (a new one is the new-module take-in's, new_modules.go).
|
||||
held func(ctx context.Context, module string) (bool, error)
|
||||
// trunk is the indexer's trunk value of the module.
|
||||
trunk func(ctx context.Context, module string) (sdkdelivery.Module, error)
|
||||
// registeredAt says the module is registered at the commit.
|
||||
registeredAt func(ctx context.Context, module, commit string) (bool, error)
|
||||
// takeIn registers the module from the builder's done key of the trunk's fingerprint (takeInKey); errKeyWaits when
|
||||
// the key is not done yet.
|
||||
takeIn func(ctx context.Context, module string, trunk sdkdelivery.Module) error
|
||||
// nodesOf are the nodes the module is assigned to; busHolder the node holding the bus, "" for none.
|
||||
nodesOf func(ctx context.Context, module string) ([]string, error)
|
||||
busHolder func(ctx context.Context) (string, error)
|
||||
// compose writes the nodes' compositions; grants writes the grants issued.
|
||||
compose func(ctx context.Context, nodes []string) error
|
||||
grants func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// take takes module's done key in and composes what it changed. It answers what it did in a line, "" when there was
|
||||
// nothing to do.
|
||||
func (s doneKeySteps) take(ctx context.Context, module string) (string, error) {
|
||||
held, err := s.held(ctx, module)
|
||||
if err != nil || !held {
|
||||
return "", err
|
||||
}
|
||||
trunk, err := s.trunk(ctx, module)
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !strings.HasPrefix(trunk.FP2, sdkdelivery.SourcePrefix2) {
|
||||
return "", nil
|
||||
}
|
||||
at, err := s.registeredAt(ctx, module, trunk.Commit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
said := ""
|
||||
if !at {
|
||||
if err := s.takeIn(ctx, module, trunk); err != nil {
|
||||
return "", err
|
||||
}
|
||||
said = fmt.Sprintf("%s registered from the builder's done key %s (%s)", module, short(trunk.FP2), short(trunk.Commit))
|
||||
}
|
||||
nodes, err := s.nodesOf(ctx, module)
|
||||
if err != nil {
|
||||
return said, err
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return said, nil
|
||||
}
|
||||
// The module's nodes first: composing them makes what the new manifest needs, its credentials among it.
|
||||
if err := s.compose(ctx, nodes); err != nil {
|
||||
return said, err
|
||||
}
|
||||
// Then the bus's user list, which carries the module's permissions at its new fingerprint.
|
||||
holder, err := s.busHolder(ctx)
|
||||
if err != nil {
|
||||
return said, err
|
||||
}
|
||||
if holder != "" && !inList(nodes, holder) {
|
||||
if err := s.compose(ctx, []string{holder}); err != nil {
|
||||
return said, err
|
||||
}
|
||||
}
|
||||
// Then the grant, which with the composition lets a want name the fingerprint (rules 6 and 7).
|
||||
if err := s.grants(ctx); err != nil {
|
||||
return said, err
|
||||
}
|
||||
if said == "" {
|
||||
said = fmt.Sprintf("%s at %s", module, short(trunk.FP2))
|
||||
}
|
||||
return said + "; composed for " + strings.Join(nodes, ", ") + ", its grant written", nil
|
||||
}
|
||||
|
||||
func inList(list []string, s string) bool {
|
||||
for _, v := range list {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// doneKeysMu keeps one take-in of a module's done key at a time in this process, per module.
|
||||
var doneKeysMu sync.Map
|
||||
|
||||
// takeDoneKey is a woken module's take-in on the live stores and buckets, said. A take-in that fails (its compositions
|
||||
// or its grant not written) is tried again every doneKeyRetry while it fails, loudly: printed each time, and an urgent
|
||||
// condition raised by the first until it has worked, as the chain's own moves are (layBuildsOutSoon). A key not done
|
||||
// yet is no failure: its next write wakes this again.
|
||||
func takeDoneKey(ctx context.Context, open *stores, module string) {
|
||||
mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
|
||||
mu.(*sync.Mutex).Lock()
|
||||
defer mu.(*sync.Mutex).Unlock()
|
||||
failed := false
|
||||
for {
|
||||
err := takeDoneKeyOnce(ctx, open, module)
|
||||
o := doneKeyObservation(module)
|
||||
if err == nil || errors.Is(err, errKeyWaits) {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "composed and granted")
|
||||
}
|
||||
return
|
||||
}
|
||||
fmt.Printf("done keys: %s: %v; tried again in %s\n", module, err, doneKeyRetry)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(doneKeyRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// doneKeyRetry is how long a failed take-in of a done key waits before it tries again.
|
||||
var doneKeyRetry = 30 * time.Second
|
||||
|
||||
// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
|
||||
// they are, no want can name it.
|
||||
const kindNotComposed = "fingerprint-not-composed"
|
||||
|
||||
func doneKeyObservation(module string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeBuild, ID: module, Token: kindNotComposed, Kind: kindNotComposed,
|
||||
Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
|
||||
Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
|
||||
}
|
||||
|
||||
// takeDoneKeyOnce is one take-in of a module's done key.
|
||||
func takeDoneKeyOnce(ctx context.Context, open *stores, module string) error {
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the builder's keys cannot be read: %w", err)
|
||||
}
|
||||
defer closeKeys()
|
||||
said, err := liveDoneKeySteps(open, keys).take(ctx, module)
|
||||
if err == nil && said != "" {
|
||||
fmt.Printf("done keys: %s\n", said)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// liveDoneKeySteps are the steps on the controller's own stores and the bus.
|
||||
func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
|
||||
inv := open.inventory
|
||||
return doneKeySteps{
|
||||
held: func(ctx context.Context, module string) (bool, error) {
|
||||
_, err := inv.SourceOf(ctx, module)
|
||||
if errors.Is(err, inventory.ErrNoSuchModule) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
},
|
||||
trunk: func(ctx context.Context, module string) (sdkdelivery.Module, error) {
|
||||
return keys.ModuleAt(ctx, module, "")
|
||||
},
|
||||
registeredAt: func(ctx context.Context, module, commit string) (bool, error) {
|
||||
src, err := inv.SourceOf(ctx, module)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return sameCommitID(src.BuiltFrom, commit), nil
|
||||
},
|
||||
takeIn: func(ctx context.Context, module string, trunk sdkdelivery.Module) error {
|
||||
_, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
},
|
||||
nodesOf: func(ctx context.Context, module string) ([]string, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == module {
|
||||
return e.On, nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
busHolder: func(ctx context.Context) (string, error) {
|
||||
holders, err := seatHolders(ctx, inv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return holders[catalogue.BrokerSeat].Node, nil
|
||||
},
|
||||
compose: func(ctx context.Context, nodes []string) error {
|
||||
_, err := composeModules(ctx, open, nodes)
|
||||
return err
|
||||
},
|
||||
grants: func(ctx context.Context) error {
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return grantsIssued(ctx, inv, js, "done keys: ")
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// watchDoneKeys wakes the take-in of a module's done key for each module the builder or the indexer writes, until ctx
|
||||
// ends: never on the watch's own goroutine.
|
||||
func watchDoneKeys(ctx context.Context, open *stores) {
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
fmt.Printf("done keys: the builder's keys are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
if err := builderkeys.Watch(ctx, keys.JS, func(module string) {
|
||||
inBackground(func() { takeDoneKey(ctx, open, module) })
|
||||
}); err != nil {
|
||||
closeKeys()
|
||||
fmt.Printf("done keys: the builder's keys are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
closeKeys()
|
||||
}()
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
)
|
||||
|
||||
// stepsFor are doneKeySteps over a module registered at registered, whose trunk value names fp at commit, assigned to
|
||||
// nodes, the bus held on holder; every act is written to acts in order.
|
||||
func stepsFor(acts *[]string, registered, commit, fp string, nodes []string, holder string) doneKeySteps {
|
||||
return doneKeySteps{
|
||||
held: func(_ context.Context, module string) (bool, error) { return module == "mesh-mail", nil },
|
||||
trunk: func(context.Context, string) (sdkdelivery.Module, error) {
|
||||
if fp == "" {
|
||||
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
||||
}
|
||||
return sdkdelivery.Module{Commit: commit, FP2: fp}, nil
|
||||
},
|
||||
registeredAt: func(_ context.Context, _ string, c string) (bool, error) { return c == registered, nil },
|
||||
takeIn: func(_ context.Context, module string, trunk sdkdelivery.Module) error {
|
||||
*acts = append(*acts, "take in "+module+" at "+trunk.Commit)
|
||||
return nil
|
||||
},
|
||||
nodesOf: func(context.Context, string) ([]string, error) { return nodes, nil },
|
||||
busHolder: func(context.Context) (string, error) { return holder, nil },
|
||||
compose: func(_ context.Context, ns []string) error {
|
||||
*acts = append(*acts, "compose "+strings.Join(ns, ","))
|
||||
return nil
|
||||
},
|
||||
grants: func(context.Context) error { *acts = append(*acts, "grants"); return nil },
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC3 (unit half): a done key of a new fingerprint of an assigned module is taken in, then the module's nodes are
|
||||
// composed, then the node holding the bus (its user list), then the grants: the grant and the composition exist
|
||||
// before the assigner's rules let a want name the fingerprint. A module registered at the trunk's commit already is
|
||||
// composed and granted without a second take-in; one assigned nowhere, unregistered, or with no trunk value is left
|
||||
// alone.
|
||||
func TestADoneKeyIsTakenInComposedAndGranted(t *testing.T) {
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
|
||||
var acts []string
|
||||
said, err := stepsFor(&acts, "old", "new", fp, []string{"ace", "g14"}, "novox").take(context.Background(), "mesh-mail")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(acts, "; "); got != "take in mesh-mail at new; compose ace,g14; compose novox; grants" {
|
||||
t.Errorf("the acts were %q", got)
|
||||
}
|
||||
if !strings.Contains(said, "registered from the builder's done key") {
|
||||
t.Errorf("said %q", said)
|
||||
}
|
||||
|
||||
acts = nil
|
||||
if _, err := stepsFor(&acts, "new", "new", fp, []string{"novox"}, "novox").take(context.Background(), "mesh-mail"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(acts, "; "); got != "compose novox; grants" {
|
||||
t.Errorf("a module registered at its trunk already: %q", got)
|
||||
}
|
||||
|
||||
for name, s := range map[string]doneKeySteps{
|
||||
"assigned nowhere": stepsFor(&acts, "old", "new", fp, nil, "novox"),
|
||||
"no trunk value": stepsFor(&acts, "old", "new", "", []string{"ace"}, "novox"),
|
||||
} {
|
||||
acts = nil
|
||||
if _, err := s.take(context.Background(), "mesh-mail"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if name == "assigned nowhere" && strings.Join(acts, "; ") != "take in mesh-mail at new" {
|
||||
t.Errorf("%s: %v", name, acts)
|
||||
}
|
||||
if name == "no trunk value" && len(acts) != 0 {
|
||||
t.Errorf("%s: %v", name, acts)
|
||||
}
|
||||
}
|
||||
acts = nil
|
||||
if _, err := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox").take(context.Background(), "unknown"); err != nil || len(acts) != 0 {
|
||||
t.Errorf("a module the catalogue does not hold: %v %v", acts, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC4: the bus's user list is a file of the bus module's composition (ADR 0539 clause 7): a change of the composed
|
||||
// users is a new revision of that composition alone, every other composition of the node keeping its digest.
|
||||
func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testing.T) {
|
||||
body, fps := capturedBody(t, "g14")
|
||||
fps["nats"] = sdkdelivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||
withUsers := func(users string) map[string]sdkdelivery.Composed {
|
||||
var d map[string]any
|
||||
if err := json.Unmarshal(body, &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// As the composer writes it for the module holding mesh-broker (mesh-sdk catalogue: BusUsersID, owned by the module).
|
||||
d["resources"] = append(d["resources"].([]any), map[string]any{"id": "nats.bus-users", "type": "file",
|
||||
"module": "nats", "path": "/var/lib/mesh/nats/users.conf", "content": users, "mode": "0600"})
|
||||
raw, _ := json.Marshal(d)
|
||||
comps, _, err := compositionsOf("g14", raw, fps, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return comps
|
||||
}
|
||||
before, after := withUsers("authorization { users = [ {user: a} ] }\n"), withUsers("authorization { users = [ {user: a}, {user: b} ] }\n")
|
||||
natsKey, _ := sdkdelivery.ComposedKey("g14", "nats", fps["nats"])
|
||||
for key, c := range before {
|
||||
d1, _ := sdkdelivery.ComposedDigestOf(c)
|
||||
d2, _ := sdkdelivery.ComposedDigestOf(after[key])
|
||||
if changed := d1 != d2; changed != (key == natsKey) {
|
||||
t.Errorf("%s changed %v; only the bus module's composition changes with its user list", key, changed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945 (the review's D1): a grant that is not written is the take-in's error, never swallowed: with no send left,
|
||||
// nothing else would write it, and no want could name the fingerprint.
|
||||
func TestAGrantNotWrittenIsTheTakeInsError(t *testing.T) {
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
|
||||
var acts []string
|
||||
s := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox")
|
||||
s.grants = func(context.Context) error { return errors.New("the bucket mesh-controller_grants: not found") }
|
||||
if _, err := s.take(context.Background(), "mesh-mail"); err == nil || !strings.Contains(err.Error(), "grants") {
|
||||
t.Errorf("a grant not written: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,362 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// composeModules composes each named node and writes its compositions that differ (ADR 0539 clause 5): what each act
|
||||
// that once sent a declaration calls instead. A node that cannot compose is that node's: the others are written, and
|
||||
// every refusal is named in the error. It answers the keys it wrote, by node. Under the lease only: a controller that
|
||||
// does not hold it writes nothing and says so.
|
||||
func composeModules(ctx context.Context, open *stores, nodes []string) (map[string][]string, error) {
|
||||
if _, err := theLease.epoch(ctx); err != nil {
|
||||
return nil, fmt.Errorf("nothing is composed: %w", err)
|
||||
}
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nothing is composed: the bus: %w", err)
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sign, err := controllerSigner(ctx, open)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nothing is composed: the controller's signing key: %w", err)
|
||||
}
|
||||
fps, err := currentFingerprints(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manifests := make(map[string]catalogue.Manifest, len(entries))
|
||||
for _, e := range entries {
|
||||
manifests[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
upgrades, err := open.inventory.Upgrades(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
policy := func(module string) (string, string, string) {
|
||||
u, ok := upgrades[module]
|
||||
if !ok {
|
||||
return "", "", ""
|
||||
}
|
||||
return u.Policy(), u.From, u.Why
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
out := map[string][]string{}
|
||||
var errs []error
|
||||
for _, node := range nodes {
|
||||
at, err := openComposedPlace(ctx, js, bus.Conn(), node)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||
continue
|
||||
}
|
||||
at.sign, at.policy = sign, policy
|
||||
if composeTrace != nil {
|
||||
traceInput(ctx, open, node)
|
||||
}
|
||||
declared, err := composeSend(ctx, open, node)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||
continue
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||
continue
|
||||
}
|
||||
// The provisioner's plans of the modules the node's composition hands to it, for the assignments mirror.
|
||||
if err := open.inventory.RecordComposedPlans(ctx, node, plansKept(declared)); err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: its provisioner's plans: %w", node, err))
|
||||
}
|
||||
comps, moved, err := compositionsOf(node, body, fps, manifests, declared.modules)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||
continue
|
||||
}
|
||||
if len(moved) > 0 {
|
||||
fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
|
||||
node, strings.Join(moved, ", "))
|
||||
}
|
||||
if composeTrace != nil {
|
||||
traceComposed(ctx, at, node, comps)
|
||||
}
|
||||
wrote, err := writeNodeCompositions(ctx, at, comps)
|
||||
if len(wrote) > 0 {
|
||||
out[node] = wrote
|
||||
fmt.Printf("compositions: %s: wrote %s\n", node, strings.Join(wrote, ", "))
|
||||
}
|
||||
// And where a data-keeping provision bound each consumer: what the next resolution keeps it at (novox/hq ADR 0232).
|
||||
if len(declared.Bindings) > 0 {
|
||||
if err := open.inventory.RecordBindings(ctx, node, declared.Bindings); err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: where its consumers are bound to their data: %w", node, err))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", node, err))
|
||||
}
|
||||
}
|
||||
return out, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// composeAndSay is composeModules for an act that has nothing to do with its answer but say it: what each former send
|
||||
// of a verb or an event calls.
|
||||
func composeAndSay(ctx context.Context, open *stores, nodes []string) error {
|
||||
if len(nodes) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := composeModules(ctx, open, nodes)
|
||||
if err != nil {
|
||||
fmt.Printf("compositions: %v\n", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// openComposedPlace opens a node's buckets: its compositions, the digests, its files, and reads its want and have.
|
||||
func openComposedPlace(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, node string) (composedPlace, error) {
|
||||
if err := broker.EnsureComposed(ctx, js, node); err != nil {
|
||||
return composedPlace{}, err
|
||||
}
|
||||
name, err := sdkdelivery.ComposedBucketFor(node)
|
||||
if err != nil {
|
||||
return composedPlace{}, err
|
||||
}
|
||||
bucket, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return composedPlace{}, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
|
||||
if err != nil {
|
||||
return composedPlace{}, fmt.Errorf("the bucket %s: %w", sdkdelivery.ControllerComposedBucket, err)
|
||||
}
|
||||
objects, err := openBucket(ctx, conn, node)
|
||||
if err != nil {
|
||||
return composedPlace{}, err
|
||||
}
|
||||
return composedPlace{node: node, bucket: bucket, digests: digests, objects: objects,
|
||||
wanted: func(ctx context.Context, module string) (string, error) { return wantedOn(ctx, js, node, module) },
|
||||
ran: func(ctx context.Context, module string) (sdkdelivery.Have, bool, error) {
|
||||
return haveOn(ctx, js, node, module)
|
||||
},
|
||||
lastApplied: lastAppliedOn(js, node),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// wantedOn is the fingerprint the assigner's want names for a module on a node; "" when there is none or the bucket is
|
||||
// not there yet.
|
||||
func wantedOn(ctx context.Context, js jetstream.JetStream, node, module string) (string, error) {
|
||||
kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key, err := sdkdelivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
w, err := sdkdelivery.DecodeWant(e.Value())
|
||||
if err != nil {
|
||||
return "", nil // a want of release 5, naming no composition: no fingerprint a composition must be kept for
|
||||
}
|
||||
return w.FP, nil
|
||||
}
|
||||
|
||||
// haveOn is what a node's node-engine says it runs of a module.
|
||||
func haveOn(ctx context.Context, js jetstream.JetStream, node, module string) (sdkdelivery.Have, bool, error) {
|
||||
kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return sdkdelivery.Have{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return sdkdelivery.Have{}, false, err
|
||||
}
|
||||
key, err := sdkdelivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
return sdkdelivery.Have{}, false, err
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return sdkdelivery.Have{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return sdkdelivery.Have{}, false, err
|
||||
}
|
||||
h, err := sdkdelivery.DecodeHave(e.Value())
|
||||
if err != nil {
|
||||
return sdkdelivery.Have{}, false, nil
|
||||
}
|
||||
return h, true, nil
|
||||
}
|
||||
|
||||
// wantedOnTheBus is every fingerprint a want names, as `<module> <fingerprint>`: what the keep set keeps the builds of
|
||||
// (ADR 0539 clause 1). Read from the assigner's bucket on each look; no bucket yet names none.
|
||||
func wantedOnTheBus(ctx context.Context) (map[string]bool, error) {
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return map[string]bool{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return map[string]bool{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for key := range lister.Keys() {
|
||||
_, module, err := sdkdelivery.ParseWantKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
w, err := sdkdelivery.DecodeWant(e.Value())
|
||||
if err != nil || w.FP == "" {
|
||||
continue
|
||||
}
|
||||
out[module+" "+w.FP] = true
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// plansKept is a composition's provisioner's plans as the store keeps them: by module, each plan and what requires a
|
||||
// module only pulled in by another (design 57 P6, issue 765); nil when it hands none.
|
||||
func plansKept(d sendable) []byte {
|
||||
if len(d.plans) == 0 {
|
||||
return nil
|
||||
}
|
||||
kept := map[string]composedPlan{}
|
||||
for module, p := range d.plans {
|
||||
kept[module] = composedPlan{Plan: p, RequiredBy: d.requiredBy[module]}
|
||||
}
|
||||
raw, _ := json.Marshal(kept) // plain strings and slices: it marshals
|
||||
return raw
|
||||
}
|
||||
|
||||
// composedPlan is one handed module's plan as the store keeps it, the shape inventory.SentProvisioningPlans reads.
|
||||
type composedPlan struct {
|
||||
catalogue.Plan
|
||||
RequiredBy string `json:"required-by,omitempty"`
|
||||
}
|
||||
|
||||
// lastAppliedOn reads, once, the declaration the node was last sent from the kept stream of declarations (the last
|
||||
// message on `mesh.node.<node>.declare`), and answers each module's slice of it (its resources that say the module, or
|
||||
// whose id names it) with the declaration's digest. None when the stream or the node's message is gone.
|
||||
func lastAppliedOn(js jetstream.JetStream, node string) func(ctx context.Context, module string) ([]json.RawMessage, string, bool, error) {
|
||||
var once sync.Once
|
||||
var last []byte
|
||||
var found bool
|
||||
var readErr error
|
||||
return func(ctx context.Context, module string) ([]json.RawMessage, string, bool, error) {
|
||||
once.Do(func() { last, found, readErr = lastDeclaration(ctx, js, node) })
|
||||
if readErr != nil || !found {
|
||||
return nil, "", false, readErr
|
||||
}
|
||||
return sliceOfDeclaration(last, module)
|
||||
}
|
||||
}
|
||||
|
||||
// lastDeclaration is the node's last signed declaration on the kept stream, as the stream holds it.
|
||||
func lastDeclaration(ctx context.Context, js jetstream.JetStream, node string) ([]byte, bool, error) {
|
||||
stream, err := js.Stream(ctx, broker.DeclarationsStream)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
msg, err := stream.GetLastMsgForSubject(ctx, "mesh.node."+node+".declare")
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return msg.Data, true, nil
|
||||
}
|
||||
|
||||
// sliceOfDeclaration is module's slice of a signed declaration as the stream holds it, and the digest of the
|
||||
// declaration it was signed over.
|
||||
func sliceOfDeclaration(data []byte, module string) ([]json.RawMessage, string, bool, error) {
|
||||
var signed link.Signed
|
||||
if err := json.Unmarshal(data, &signed); err != nil {
|
||||
return nil, "", false, fmt.Errorf("the last declaration is not readable: %w", err)
|
||||
}
|
||||
var envelope struct {
|
||||
Resources []json.RawMessage `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(signed.Declaration, &envelope); err != nil {
|
||||
return nil, "", false, fmt.Errorf("the last declaration is not readable: %w", err)
|
||||
}
|
||||
sum := sha256.Sum256(signed.Declaration)
|
||||
return sliceOfModule(envelope.Resources, module), "sha256:" + hex.EncodeToString(sum[:]), true, nil
|
||||
}
|
||||
|
||||
// sliceOfModule is, of a declaration's resources in their order, those of module: saying it, or with no module said
|
||||
// and an id that names it (sdkdecl.ModuleGuess).
|
||||
func sliceOfModule(resources []json.RawMessage, module string) []json.RawMessage {
|
||||
var out []json.RawMessage
|
||||
for _, r := range resources {
|
||||
var head struct {
|
||||
ID string `json:"id"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
if json.Unmarshal(r, &head) != nil {
|
||||
continue
|
||||
}
|
||||
owner := head.Module
|
||||
if owner == "" {
|
||||
owner, _ = sdkdecl.ModuleGuess(head.ID)
|
||||
}
|
||||
if owner == module {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1945/AC13: the node-engine's composition on a node carries the operator's public keys as its file operator-keys,
|
||||
// from its setting operator-signers, beside the controller's key it already trusts (ADR 0539 clause 3, decided with the
|
||||
// lead 2026-10-24: the switch's last declaration carries it because today's controller composes the same file from
|
||||
// the same manifest). The manifest is mesh-host's module.json at 20d0998c (testdata/mesh-host-module.json), never
|
||||
// typed by hand.
|
||||
func TestTheNodeEnginesCompositionCarriesTheOperatorsKeys(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/mesh-host-module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its archive names a build artifact, which only its registration turns into a reference: left out here, the rest of
|
||||
// the manifest as captured.
|
||||
var kept []map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "archive" {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
m.Resources = kept
|
||||
const key = "MCowBQYDK2VwAyEAb3BlcmF0b3Itc2lnbmluZy1rZXktZm9yLXRlc3RzLW9ubHk="
|
||||
in := compose.Input{Catalogue: map[string]catalogue.Manifest{m.Module: m},
|
||||
Nodes: []compose.Node{{Name: "ace", Assigned: []string{m.Module}, Stated: true,
|
||||
Settings: map[string][]catalogue.Layer{m.Module: {{From: "mesh", Values: map[string]any{"operator-signers": key}}}}}}}
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64)
|
||||
all, _, err := compose.Compositions(in, "ace", map[string]string{m.Module: fp}, compose.Reading)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := sdkdelivery.ComposedKey("ace", m.Module, fp)
|
||||
var file struct{ Path, Content string }
|
||||
for _, r := range all[k].Resources {
|
||||
var head struct{ ID, Path, Content string }
|
||||
_ = json.Unmarshal(r, &head)
|
||||
if strings.HasSuffix(head.ID, "operator-keys") {
|
||||
file.Path, file.Content = head.Path, head.Content
|
||||
}
|
||||
}
|
||||
if file.Path != "/var/lib/mesh-host/operator-keys" || file.Content != key+"\n" {
|
||||
t.Errorf("the node-engine's composition carries operator-keys at %q with %q, want the operator's key", file.Path, file.Content)
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// 1945 (dev-r6-controller's finding): the provisioner's plans a composition hands are kept in the shape the
|
||||
// assignments mirror reads (inventory.SentProvisioningPlans: each module's plan and what requires it), and a
|
||||
// composition that hands none keeps none.
|
||||
func TestAComposedNodesProvisionerPlansAreKept(t *testing.T) {
|
||||
if plansKept(sendable{}) != nil {
|
||||
t.Errorf("a composition that hands nothing kept plans")
|
||||
}
|
||||
d := sendable{plans: map[string]catalogue.Plan{"umami": {Placement: map[string]string{"dir:state": "/var/lib/umami"}}},
|
||||
requiredBy: map[string]string{"umami": "analytics"}}
|
||||
var kept map[string]struct {
|
||||
Placement map[string]string `json:"placement"`
|
||||
RequiredBy string `json:"required-by"`
|
||||
}
|
||||
if err := json.Unmarshal(plansKept(d), &kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept["umami"].Placement["dir:state"] != "/var/lib/umami" || kept["umami"].RequiredBy != "analytics" {
|
||||
t.Errorf("kept %+v", kept)
|
||||
}
|
||||
}
|
||||
@@ -1,169 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The judgement beside each composition is rewritten when what it is judged against or with changes (novox/hq ADR
|
||||
// 0539 clauses 2 and 14): when the node's have of the module changes (it runs another composition, or holds), and when
|
||||
// a person sets the module's policy. Heard as the have bucket's writes and the `upgrade` verb, never on a timer.
|
||||
|
||||
// rejudge writes again the judgement and policy beside every composition of module on the node at, where they differ.
|
||||
func rejudge(ctx context.Context, at composedPlace, module string) (int, error) {
|
||||
lister, err := at.bucket.ListKeys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
wrote := 0
|
||||
var errs []error
|
||||
for key := range lister.Keys() {
|
||||
node, m, _, isNode, err := sdkdelivery.ParseComposedKey(key)
|
||||
if err != nil || isNode || node != at.node || m != module {
|
||||
continue
|
||||
}
|
||||
c, revision, err := readComposed(ctx, at.bucket, key)
|
||||
if err != nil || revision == 0 {
|
||||
continue
|
||||
}
|
||||
c.Signature = nil
|
||||
digest, err := sdkdelivery.ComposedDigestOf(c)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
judged, err := judgeMove(ctx, at, key, c, digest)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
w, err := keepDigest(ctx, at.digests, key, judged)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
if w {
|
||||
wrote++
|
||||
}
|
||||
}
|
||||
return wrote, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// rejudgeOn opens the node's buckets and rejudges module there, under the lease.
|
||||
func rejudgeOn(ctx context.Context, open *stores, nodes []string, module string) error {
|
||||
if _, err := theLease.epoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
upgrades, err := open.inventory.Upgrades(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, node := range nodes {
|
||||
at, err := openComposedPlace(ctx, js, bus.Conn(), node)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
at.policy = func(m string) (string, string, string) {
|
||||
u, ok := upgrades[m]
|
||||
if !ok {
|
||||
return "", "", ""
|
||||
}
|
||||
return u.Policy(), u.From, u.Why
|
||||
}
|
||||
if _, err := rejudge(ctx, at, module); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// rejudgeModule rejudges a module on every node it is assigned to: what a person's policy changes.
|
||||
func rejudgeModule(ctx context.Context, open *stores, module string) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err)
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == module && len(e.On) > 0 {
|
||||
if err := rejudgeOn(ctx, open, e.On, module); err != nil {
|
||||
fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watchHavesForJudgement rejudges a node's module each time its have changes, until ctx ends: never on the watch's own
|
||||
// goroutine.
|
||||
func watchHavesForJudgement(ctx context.Context, open *stores) {
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
bus.Close()
|
||||
return
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket)
|
||||
if err != nil {
|
||||
bus.Close()
|
||||
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
bus.Close()
|
||||
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer bus.Close()
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
node, module, isNode, err := sdkdelivery.ParseHaveKey(e.Key())
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
inBackground(func() {
|
||||
if err := rejudgeOn(ctx, open, []string{node}, module); err != nil {
|
||||
fmt.Printf("compositions: %s on %s is not judged again: %v\n", module, node, err)
|
||||
}
|
||||
// A node's node-engine moved: the declarations' stream goes once every node runs the new one.
|
||||
if module == nodeEngineModule {
|
||||
if _, err := retireDeclarations(ctx, open); err != nil {
|
||||
fmt.Printf("compositions: whether the declarations' stream may go cannot be read: %v\n", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -1,582 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The compositions through the controller whole (novox/hq ADR 0539; the review's G1): the real store, the controller's
|
||||
// signing key, and a bus that runs the user list the controller composes from that store, so a subject release 6 uses
|
||||
// and the list does not grant is refused as it would be on the mesh. serve() runs whole on it; the builder and the
|
||||
// indexer are their own accounts writing their own keys; a rotation is the operator's verb.
|
||||
type servedMesh struct {
|
||||
open *stores
|
||||
controller jetstream.JetStream
|
||||
builder jetstream.JetStream
|
||||
indexer jetstream.JetStream
|
||||
}
|
||||
|
||||
// keyOfManifest is a done key of a module at fp with no artifacts, its manifest the given one, and the indexer's trunk
|
||||
// value naming it in the catalogue's repository at commit.
|
||||
func keyOfManifest(t *testing.T, m catalogue.Manifest, fp, commit string) (raw []byte, k sdkdelivery.BuilderBuild, trunk sdkdelivery.Module) {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := sdkdelivery.ManifestDigest(raw)
|
||||
return raw, sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "anchor", Manifest: digest},
|
||||
sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: commit, Path: "modules/" + m.Module}
|
||||
}
|
||||
|
||||
func aServedMesh(t *testing.T) *servedMesh {
|
||||
t.Helper()
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withTrunkList(t, true)
|
||||
ident := identity.ForTest(t)
|
||||
if _, err := ident.Establish(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The bus, running the user list the controller composes from the store, with the builder and the indexer as the
|
||||
// accounts their own manifests make (each writes its own state).
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var controller broker.Principal
|
||||
for _, p := range users {
|
||||
if p.Kind == broker.KindController {
|
||||
controller = p
|
||||
}
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: controller.Username(), Kind: inventory.BusController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builder := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.BuilderModule, State: []string{"builds"}}
|
||||
indexer := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: sdkdelivery.IndexerModule, State: []string{"modules", "manifests"}}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hashes := map[string]string{}
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
fixture, _ := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
hashes[builder.Username()], hashes[indexer.Username()] = string(fixture), string(fixture)
|
||||
filled, missing := broker.WithPasswords([]broker.Principal{controller, builder, indexer}, hashes)
|
||||
if len(missing) > 0 {
|
||||
t.Fatalf("no password for %v", missing)
|
||||
}
|
||||
accounts, err := broker.ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed user list does not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
u, _ := url.Parse(s.ClientURL())
|
||||
u.User = url.UserPassword(controller.Username(), password)
|
||||
t.Setenv(broker.NATSVar, u.String())
|
||||
|
||||
connect := func(user, pw string) jetstream.JetStream {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, pw), nats.CustomInboxPrefix("_INBOX."+user))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
m := &servedMesh{open: open, controller: connect(controller.Username(), password),
|
||||
builder: connect(builder.Username(), "pw"), indexer: connect(indexer.Username(), "pw")}
|
||||
|
||||
// The test's own inventory on the bus's chain, as the serving controller's will be: what is registered and assigned
|
||||
// below is in the chain it reads (assignments and builds live on the bus, ADR 0408).
|
||||
chainConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(controller.Username(), password),
|
||||
nats.CustomInboxPrefix("_INBOX."+controller.Username()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(chainConn.Close)
|
||||
// The controller's own buckets, the chain's among them, as it asserts them at its start.
|
||||
asserter, err := broker.Dial(u.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(asserter.Close)
|
||||
if err := asserter.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatalf("the controller's buckets: %v", err)
|
||||
}
|
||||
c, err := chain.OnTheBus(ctx, chainConn, func(context.Context) error { return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.UseChain(c)
|
||||
|
||||
// A provider and its consumer (bindings_test.go's store and board), registered from done keys of the catalogue's
|
||||
// repository, so each has a fingerprint of the new form; the store holds the mesh's seat on anchor, the board on
|
||||
// laptop is bound to it.
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, m := range storeManifests() {
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
// The board is given its credential in a file, as a consumer of a database is.
|
||||
board := shelf["board"]
|
||||
board.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"}
|
||||
shelf["board"] = board
|
||||
for i, module := range []string{"store", "board"} {
|
||||
raw, k, trunk := keyOfManifest(t, shelf[module], fpOf(byte('a'+i)), "c0ffee0123456789")
|
||||
keys := trunkKeys{keyBuckets: keyBuckets{keys: map[string]builderkeys.Key{module: aBuilderKey(module, trunk.FP2, k)},
|
||||
manifests: map[string]string{trunk.Manifest: string(raw)}}, trunk: trunk}
|
||||
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
|
||||
t.Fatalf("%s is not registered from its key: %v", module, err)
|
||||
}
|
||||
}
|
||||
// The private network on both machines, as aMesh assigns it, now on the chain the controller reads.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "laptop", "board"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The builder's and the indexer's buckets as their own parts make them, there before the controller starts (its
|
||||
// watch of their keys begins at its start); and the plans' buckets a controller of release 5 left.
|
||||
for _, b := range []string{sdkdelivery.BuilderBuildsBucket, sdkdelivery.ModulesBucket, sdkdelivery.ManifestsBucket} {
|
||||
if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 5}); err != nil {
|
||||
t.Fatalf("the bucket %s: %v", b, err)
|
||||
}
|
||||
}
|
||||
for _, b := range broker.PlanStateBuckets {
|
||||
if _, err := m.controller.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- serve(serving) }()
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Log("the controller did not stop within 30 s")
|
||||
}
|
||||
})
|
||||
// Served once both nodes have their own composition: the start's pass ran.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
own, _ := sdkdelivery.ComposedNodeKey(node)
|
||||
m.waitFor(t, done, node, own, 0)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// aBuilderKey is the builder's done key of module at fp, as the take-in reads it.
|
||||
func aBuilderKey(module, fp string, b sdkdelivery.BuilderBuild) builderkeys.Key {
|
||||
return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, Build: b}
|
||||
}
|
||||
|
||||
// waitFor waits until the node's composition bucket holds key at a revision above after, failing when the controller
|
||||
// stops or a minute passes: a test's wait for what an event made, never a loop of the code under test.
|
||||
func (m *servedMesh) waitFor(t *testing.T, done chan error, node, key string, after uint64) uint64 {
|
||||
t.Helper()
|
||||
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("the controller stopped: %v", err)
|
||||
default:
|
||||
}
|
||||
if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil {
|
||||
if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after {
|
||||
return e.Revision()
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("%s in %s was not written within a minute; %s", key, bucket, m.whatIsThere(t, node))
|
||||
return 0
|
||||
}
|
||||
|
||||
// whatIsThere says, for a failure, what the node's bucket holds, the ids its own composition carries and the
|
||||
// fingerprints the controller has registered.
|
||||
func (m *servedMesh) whatIsThere(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||
out := "keys:"
|
||||
kv, err := m.controller.KeyValue(context.Background(), bucket)
|
||||
if err != nil {
|
||||
return out + " " + err.Error()
|
||||
}
|
||||
lister, err := kv.ListKeys(context.Background())
|
||||
if err == nil {
|
||||
for k := range lister.Keys() {
|
||||
out += " " + k
|
||||
}
|
||||
}
|
||||
if lister, err := kv.ListKeys(context.Background()); err == nil {
|
||||
for k := range lister.Keys() {
|
||||
if e, err := kv.Get(context.Background(), k); err == nil {
|
||||
if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil {
|
||||
out += fmt.Sprintf("; %s at revision %d carries", k, e.Revision())
|
||||
for _, r := range c.Resources {
|
||||
var head struct{ ID string }
|
||||
_ = json.Unmarshal(r, &head)
|
||||
out += " " + head.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
holders, herr := m.open.inventory.HoldersOf(context.Background(), "postgres-database", "")
|
||||
out += fmt.Sprintf("; holders of postgres-database: %+v (%v)", holders, herr)
|
||||
own, _ := sdkdelivery.ComposedNodeKey(node)
|
||||
if e, err := kv.Get(context.Background(), own); err == nil {
|
||||
if c, err := sdkdelivery.DecodeComposed(e.Value()); err == nil {
|
||||
out += "; its own carries:"
|
||||
for _, r := range c.Resources {
|
||||
var head struct{ ID string }
|
||||
_ = json.Unmarshal(r, &head)
|
||||
out += " " + head.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
fps, err := currentFingerprints(context.Background(), m.open.inventory)
|
||||
out += fmt.Sprintf("; fingerprints registered: %v (%v)", fps, err)
|
||||
// And what composing the node says now, in this process, under the serving controller's lease.
|
||||
_, cerr := composeModules(context.Background(), m.open, []string{node})
|
||||
return out + fmt.Sprintf("; composing %s now: %v", node, cerr)
|
||||
}
|
||||
|
||||
// revisions are a node's composition keys and their revisions.
|
||||
func (m *servedMesh) revisions(t *testing.T, node string) map[string]uint64 {
|
||||
t.Helper()
|
||||
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||
kv, err := m.controller.KeyValue(t.Context(), bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lister, err := kv.ListKeys(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]uint64{}
|
||||
for k := range lister.Keys() {
|
||||
if e, err := kv.Get(t.Context(), k); err == nil {
|
||||
out[k] = e.Revision()
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declarationsSent is how many declarations the bus's stream of them holds: none is ever sent in release 6.
|
||||
func (m *servedMesh) declarationsSent(t *testing.T) uint64 {
|
||||
t.Helper()
|
||||
s, err := m.controller.Stream(t.Context(), broker.DeclarationsStream)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return 0
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := s.Info(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.State.Msgs
|
||||
}
|
||||
|
||||
// 1945/AC6, AC14: started whole on a bus running its own user list, the controller composes every node and every
|
||||
// assigned module, deletes the plans' buckets and says each as bucket-deleted, and sends no declaration.
|
||||
func TestServedTheStartComposesEveryNodeAndDeletesThePlansBuckets(t *testing.T) {
|
||||
m := aServedMesh(t)
|
||||
for _, k := range []struct{ node, module string }{{"anchor", "store"}, {"laptop", "board"}} {
|
||||
fp := map[string]string{"store": fpOf('a'), "board": fpOf('b')}[k.module]
|
||||
key, _ := sdkdelivery.ComposedKey(k.node, k.module, fp)
|
||||
m.waitFor(t, nil, k.node, key, 0)
|
||||
}
|
||||
for _, b := range broker.PlanStateBuckets {
|
||||
if _, err := m.controller.KeyValue(t.Context(), b); !errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
t.Errorf("%s is still there: %v", b, err)
|
||||
}
|
||||
}
|
||||
if n := m.declarationsSent(t); n != 0 {
|
||||
t.Errorf("%d declarations were sent", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC3: the builder writes the done key of a new fingerprint of an assigned module and the indexer its trunk
|
||||
// value: the controller, hearing the keys, registers it, writes its composition on the node it is assigned to and
|
||||
// `grants.<module>.<fp>`; nothing is sent.
|
||||
func TestServedADoneKeyIsComposedAndGranted(t *testing.T) {
|
||||
m := aServedMesh(t)
|
||||
ctx := t.Context()
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, mf := range storeManifests() {
|
||||
shelf[mf.Module] = mf
|
||||
}
|
||||
next := shelf["board"]
|
||||
next.Secrets = map[string]string{"postgres-database": "/var/lib/board/database-password"}
|
||||
next.Version = "2"
|
||||
fp := fpOf('c')
|
||||
raw, k, trunk := keyOfManifest(t, next, fp, "d00d0123456789ab")
|
||||
manifests, err := m.indexer.KeyValue(ctx, sdkdelivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, _ := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "board", JSON: string(raw)})
|
||||
if _, err := manifests.Put(ctx, trunk.Manifest, value); err != nil {
|
||||
t.Fatalf("the indexer cannot write its manifest: %v", err)
|
||||
}
|
||||
builds, err := m.builder.KeyValue(ctx, sdkdelivery.BuilderBuildsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := sdkdelivery.BuilderBuildKey("board", fp)
|
||||
value, _ = sdkdelivery.EncodeBuilderBuild(k)
|
||||
if _, err := builds.Put(ctx, key, value); err != nil {
|
||||
t.Fatalf("the builder cannot write its key: %v", err)
|
||||
}
|
||||
modules, err := m.indexer.KeyValue(ctx, sdkdelivery.ModulesBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mk, _ := sdkdelivery.ModulesKey("board")
|
||||
value, _ = sdkdelivery.EncodeModule(trunk)
|
||||
if _, err := modules.Put(ctx, mk, value); err != nil {
|
||||
t.Fatalf("the indexer cannot write its trunk value: %v", err)
|
||||
}
|
||||
composed, _ := sdkdelivery.ComposedKey("laptop", "board", fp)
|
||||
m.waitFor(t, nil, "laptop", composed, 0)
|
||||
grants, err := m.controller.KeyValue(ctx, broker.GrantsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gk, _ := sdkdelivery.ControllerGrantKey("board", fp)
|
||||
if _, err := grants.Get(ctx, gk); err != nil {
|
||||
t.Errorf("grants.%s is not written: %v", gk, err)
|
||||
}
|
||||
if n := m.declarationsSent(t); n != 0 {
|
||||
t.Errorf("%d declarations were sent", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC2: the operator rotates the store's credential, the real verb: the compositions that carry it (the store's
|
||||
// grants on anchor, and the board's on laptop when it carries the credential) get a new revision, every other
|
||||
// composition keeps its own, and nothing is sent.
|
||||
func TestServedARotationRewritesOnlyTheCompositionsThatUseIt(t *testing.T) {
|
||||
m := aServedMesh(t)
|
||||
// Everything that writes or discards a credential row, and what composing each node saw and decided, in order, with the
|
||||
// callers: one failure shows the cause (issue 1945).
|
||||
var traceMu sync.Mutex
|
||||
var trace []string
|
||||
note := func(line string) {
|
||||
traceMu.Lock()
|
||||
defer traceMu.Unlock()
|
||||
trace = append(trace, time.Now().Format("15:04:05.000")+" "+line)
|
||||
}
|
||||
inventory.SecretTrace = func(op, what string) { note("secret " + op + ": " + what) }
|
||||
composeTrace = func(line string) { note("compose " + line) }
|
||||
t.Cleanup(func() { inventory.SecretTrace, composeTrace = nil, nil })
|
||||
traced := func() string {
|
||||
traceMu.Lock()
|
||||
defer traceMu.Unlock()
|
||||
return strings.Join(trace, "\n ")
|
||||
}
|
||||
defer func() {
|
||||
if t.Failed() {
|
||||
t.Logf("TRACE\n %s", traced())
|
||||
}
|
||||
}()
|
||||
before := map[string]map[string]uint64{"anchor": m.revisions(t, "anchor"), "laptop": m.revisions(t, "laptop")}
|
||||
holders, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
|
||||
var waited uint64
|
||||
// The verb runs here as its child process runs it: its inventory takes the chain from the bus the served controller keeps it
|
||||
// on (the registrations and assignments are there), not the one in memory the fixture's store was made with.
|
||||
inventory.OnTheBusChainForTest(t)
|
||||
said, err := capturedOutput(func() error {
|
||||
if err := rotateCommand(t.Context(), []string{"postgres-database"}); err != nil {
|
||||
return err
|
||||
}
|
||||
// The serving controller's own lines while the rotation lands are captured too.
|
||||
boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b'))
|
||||
waited = m.waitForOrSay(t, "laptop", boardKey, before["laptop"][boardKey])
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("rotate: %v\n%s", err, said)
|
||||
}
|
||||
t.Logf("rotate said (holders before: %+v):\n%s", holders, said)
|
||||
defer func() {
|
||||
if t.Failed() {
|
||||
t.Logf("rotate said:\n%s", said)
|
||||
}
|
||||
}()
|
||||
boardKey, _ := sdkdelivery.ComposedKey("laptop", "board", fpOf('b'))
|
||||
storeKey, _ := sdkdelivery.ComposedKey("anchor", "store", fpOf('a'))
|
||||
// The board's sealed credential file is new: its composition moves with the rotation.
|
||||
if rev := waited; rev == 0 {
|
||||
var kept []string
|
||||
for _, line := range strings.Split(said, "\n") {
|
||||
for _, w := range []string{"board", "postgres", "secret", "credential", "laptop", "withdr", "retir", "error", "refused"} {
|
||||
if strings.Contains(line, w) {
|
||||
kept = append(kept, line)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
said = strings.Join(kept, " | ")
|
||||
after, _ := m.open.inventory.HoldersOf(t.Context(), "postgres-database", "")
|
||||
in, ierr := inputFor(t.Context(), m.open)
|
||||
var credentials string
|
||||
if ierr == nil {
|
||||
credentials = fmt.Sprintf("would make %+v; credentials %+v", in.wouldMake, in.Credentials)
|
||||
}
|
||||
state, serr := m.open.inventory.PairCredentialState(t.Context(), inventory.PairCredential{Name: "postgres-database",
|
||||
Consumer: "laptop", ConsumerModule: "board", Provider: "anchor"})
|
||||
d, derr := composeSend(t.Context(), m.open, "laptop")
|
||||
var sealed []string
|
||||
for _, r := range d.Resources {
|
||||
if r["id"] == "board.secret-postgres-database" {
|
||||
sealed = append(sealed, fmt.Sprint(r["sealed"])[:min(24, len(fmt.Sprint(r["sealed"])))])
|
||||
}
|
||||
}
|
||||
diag := fmt.Sprintf("state %+v %v; compose now %v sealed %v; %s %v", state, serr, derr, sealed, credentials, ierr)
|
||||
// One line: the verdict keeps only a failure's first line (issue 1921).
|
||||
t.Fatalf("the board's composition did not move within a minute after the rotation (revision before %d); TRACE %s; LINES %s; "+
|
||||
"holders before %+v, after %+v; DIAG %s; %s", before["laptop"][boardKey],
|
||||
strings.Join(strings.Fields(strings.ReplaceAll(traced(), "\n", " | ")), " "), strings.Join(strings.Fields(said), " "),
|
||||
holders, after, diag, strings.Join(strings.Fields(m.whatIsThere(t, "laptop")), " "))
|
||||
}
|
||||
moved := map[string]bool{}
|
||||
for node, was := range before {
|
||||
for k, rev := range m.revisions(t, node) {
|
||||
if rev != was[k] {
|
||||
moved[k] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for k := range moved {
|
||||
if k != boardKey && k != storeKey {
|
||||
t.Errorf("%s moved: only the compositions carrying the credential do", k)
|
||||
}
|
||||
}
|
||||
if !moved[boardKey] {
|
||||
t.Errorf("the board's composition did not move: %v", moved)
|
||||
}
|
||||
if n := m.declarationsSent(t); n != 0 {
|
||||
t.Errorf("%d declarations were sent", n)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForOrSay is waitFor that answers 0 instead of failing, for a caller that says more.
|
||||
func (m *servedMesh) waitForOrSay(t *testing.T, node, key string, after uint64) uint64 {
|
||||
t.Helper()
|
||||
bucket, _ := sdkdelivery.ComposedBucketFor(node)
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
if kv, err := m.controller.KeyValue(t.Context(), bucket); err == nil {
|
||||
if e, err := kv.Get(t.Context(), key); err == nil && e.Revision() > after {
|
||||
return e.Revision()
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// capturedOutput runs act with this process's standard output captured, and answers what it printed.
|
||||
func capturedOutput(act func() error) (string, error) {
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
return "", act()
|
||||
}
|
||||
was := os.Stdout
|
||||
os.Stdout = w
|
||||
done := make(chan string)
|
||||
go func() {
|
||||
var b strings.Builder
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
b.Write(buf[:n])
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
done <- b.String()
|
||||
}()
|
||||
actErr := act()
|
||||
os.Stdout = was
|
||||
w.Close()
|
||||
return <-done, actErr
|
||||
}
|
||||
@@ -1,341 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the controller does at its start under the lease (novox/hq ADR 0539 clauses 10 and 13):
|
||||
//
|
||||
// - it deletes the buckets of the plans' state this release deleted, each said once in its log and as its event
|
||||
// `bucket-deleted`: a later start finds none and says nothing;
|
||||
// - it composes every node, so every assigned module has its composition at the fingerprint it is registered at
|
||||
// and each node its own (ADR 0539 clause 11: the new controller writes every composition at its start);
|
||||
// - and it composes, on the control-node, the controller it replaced at that controller's own fingerprint, from that
|
||||
// build's manifest, so the operator's way back is one key at the terminal (clause 10).
|
||||
|
||||
// planStateBucketDeleter is what deleting the plans' buckets needs of the bus: jetstream.JetStream, or a test's.
|
||||
type planStateBucketDeleter interface {
|
||||
DeleteKeyValue(ctx context.Context, bucket string) error
|
||||
}
|
||||
|
||||
// deletePlanStateBuckets deletes each bucket of the plans' state the bus still holds and says each it deleted; it
|
||||
// answers those it deleted. A bucket not there is no error and is not said.
|
||||
func deletePlanStateBuckets(ctx context.Context, js planStateBucketDeleter, say func(bucket string)) ([]string, error) {
|
||||
var deleted []string
|
||||
var errs []error
|
||||
for _, bucket := range broker.PlanStateBuckets {
|
||||
err := js.DeleteKeyValue(ctx, bucket)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound), errors.Is(err, jetstream.ErrStreamNotFound):
|
||||
continue
|
||||
case err != nil:
|
||||
errs = append(errs, fmt.Errorf("deleting %s: %w", bucket, err))
|
||||
continue
|
||||
}
|
||||
deleted = append(deleted, bucket)
|
||||
say(bucket)
|
||||
}
|
||||
return deleted, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// BucketDeleted is the body of the controller's event `bucket-deleted`.
|
||||
type BucketDeleted struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// sayBucketDeleted says a deleted bucket in the log and as the controller's event.
|
||||
func sayBucketDeleted(ctx context.Context, bucket string) {
|
||||
sayDeleted(ctx, bucket, "a bucket of the plans' state, which release 6 deleted (novox/hq ADR 0539 clause 13)")
|
||||
}
|
||||
|
||||
// sayDeleted says a deleted bucket or stream, with why, in the log and as the controller's event bucket-deleted.
|
||||
func sayDeleted(ctx context.Context, bucket, why string) {
|
||||
fmt.Printf("start: deleted %s: %s\n", bucket, why)
|
||||
bus := checkEvents
|
||||
if bus == nil {
|
||||
return
|
||||
}
|
||||
body, err := json.Marshal(BucketDeleted{Bucket: bucket, Why: why, At: time.Now().UTC()})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
saying, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer stop()
|
||||
if err := bus.PublishSeatEvent(saying, link.MeshControllerSeat, "bucket-deleted", body); err != nil {
|
||||
fmt.Printf("start: that %s was deleted could not be said as the event bucket-deleted: %v\n", bucket, err)
|
||||
}
|
||||
}
|
||||
|
||||
// composeAtStart is the start's pass: the plans' buckets deleted, every node composed, the replaced controller
|
||||
// composed. Said, never fatal: what is not composed now is composed by the next act that touches it.
|
||||
func composeAtStart(ctx context.Context, open *stores) {
|
||||
if bus, err := aBus(); err != nil {
|
||||
fmt.Printf("start: the plans' buckets are not looked at: %v\n", err)
|
||||
} else {
|
||||
if js, err := jetstream.New(bus.Conn()); err == nil {
|
||||
if _, err := deletePlanStateBuckets(ctx, js, func(b string) { sayBucketDeleted(ctx, b) }); err != nil {
|
||||
fmt.Printf("start: %v\n", err)
|
||||
}
|
||||
}
|
||||
bus.Close()
|
||||
}
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("start: nothing is composed: %v\n", err)
|
||||
return
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
if _, err := composeModules(ctx, open, names); err != nil {
|
||||
fmt.Printf("start: %v\n", err)
|
||||
}
|
||||
if err := composeReplacedController(ctx, open); err != nil {
|
||||
fmt.Printf("start: the controller this one replaced is not composed: %v\n", err)
|
||||
}
|
||||
if _, err := retireDeclarations(ctx, open); err != nil {
|
||||
fmt.Printf("start: whether the declarations' stream may go cannot be read: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// nodeEngineModule is the node-engine's module.
|
||||
const nodeEngineModule = "mesh-host"
|
||||
|
||||
// declarationsDone says every node runs the node-engine at fingerprint current, as its have names: no node still reads
|
||||
// a declaration, and none away will need its last one (ADR 0539 clause 11). No node, or no fingerprint of the new
|
||||
// form, is not done.
|
||||
func declarationsDone(nodes []string, runs map[string]string, current string) bool {
|
||||
if len(nodes) == 0 || !strings.HasPrefix(current, sdkdelivery.SourcePrefix2) {
|
||||
return false
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if runs[n] != current {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// declarationsRetiredOn reads whether the declarations are done on the mesh now: every node's have of the node-engine
|
||||
// against the fingerprint it is registered at.
|
||||
func declarationsRetiredOn(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream) (bool, error) {
|
||||
fps, err := currentFingerprints(ctx, inv)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
names := make([]string, 0, len(nodes))
|
||||
runs := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
names = append(names, n.Name)
|
||||
h, found, err := haveOn(ctx, js, n.Name, nodeEngineModule)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if found {
|
||||
runs[n.Name] = h.FP
|
||||
}
|
||||
}
|
||||
return declarationsDone(names, runs, fps[nodeEngineModule]), nil
|
||||
}
|
||||
|
||||
// retireDeclarations deletes the declarations' stream, with every node's consumer on it, once every node runs the new
|
||||
// node-engine, and says it once; a stream already gone is said by nobody. It answers whether the declarations are done.
|
||||
func retireDeclarations(ctx context.Context, open *stores) (bool, error) {
|
||||
if _, err := theLease.epoch(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
done, err := declarationsRetiredOn(ctx, open.inventory, js)
|
||||
if err != nil || !done {
|
||||
return done, err
|
||||
}
|
||||
err = js.DeleteStream(ctx, broker.DeclarationsStream)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrStreamNotFound):
|
||||
return true, nil
|
||||
case err != nil:
|
||||
return true, fmt.Errorf("deleting the stream %s: %w", broker.DeclarationsStream, err)
|
||||
}
|
||||
sayDeleted(ctx, broker.DeclarationsStream, "the stream of each node's last declaration: every node runs the new "+
|
||||
"node-engine, which reads none (novox/hq ADR 0539 clause 11)")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// replacedController is the controller a control-node ran before this one: its fingerprint and manifest.
|
||||
type replacedController struct {
|
||||
node, fp string
|
||||
manifest catalogue.Manifest
|
||||
}
|
||||
|
||||
// replacedControllers are, for each node the controller is assigned to, the controller it replaced: the newest build of
|
||||
// the controller that worked, whose fingerprint is of the new form and not the one the controller is registered at now
|
||||
// (the assignments name no build since ADR 0539 §12, so the build records say it). None when there is no such build.
|
||||
func replacedControllers(ctx context.Context, open *stores) ([]replacedController, error) {
|
||||
inv := open.inventory
|
||||
fps, err := currentFingerprints(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var on []string
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == controllerModule {
|
||||
on = e.On
|
||||
}
|
||||
}
|
||||
if len(on) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
builds, err := inv.Builds(ctx, controllerModule, 20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
before, found := replacedBuild(builds, fps[controllerModule])
|
||||
if !found {
|
||||
return nil, nil
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(before.Manifest, &m); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s's build %s: %w", controllerModule, before.ID, err)
|
||||
}
|
||||
var out []replacedController
|
||||
for _, node := range on {
|
||||
out = append(out, replacedController{node: node, fp: before.SourceFingerprint, manifest: m})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// replacedBuild is, of a module's builds newest first, the newest that worked, kept its manifest, and names a
|
||||
// fingerprint of the new form other than current.
|
||||
func replacedBuild(builds []inventory.Build, current string) (inventory.Build, bool) {
|
||||
for _, b := range builds {
|
||||
if b.Worked() && b.SourceFingerprint != current && strings.HasPrefix(b.SourceFingerprint, sdkdelivery.SourcePrefix2) &&
|
||||
len(b.Manifest) > 0 {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return inventory.Build{}, false
|
||||
}
|
||||
|
||||
// composeReplacedController writes, on each control-node, the controller's composition at the fingerprint of the one
|
||||
// it replaced: the node composed with that build's manifest in place of the controller's registered one, in the
|
||||
// reading mode (nothing made), and only the controller's composition of it kept.
|
||||
func composeReplacedController(ctx context.Context, open *stores) error {
|
||||
if _, err := theLease.epoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
replaced, err := replacedControllers(ctx, open)
|
||||
if err != nil || len(replaced) == 0 {
|
||||
return err
|
||||
}
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sign, err := controllerSigner(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, r := range replaced {
|
||||
in, err := inputFor(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, err := replacedComposition(in.Input, r)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("%s: %w", r.node, err))
|
||||
continue
|
||||
}
|
||||
at, err := openComposedPlace(ctx, js, bus.Conn(), r.node)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
at.sign = sign
|
||||
key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp)
|
||||
if wrote, err := writeComposed(ctx, at, key, c); err != nil {
|
||||
errs = append(errs, err)
|
||||
} else if wrote {
|
||||
fmt.Printf("start: composed %s, the controller this one replaced, on %s: the way back is want.%s.%s naming it\n",
|
||||
short(r.fp), r.node, r.node, controllerModule)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// replacedComposition is the controller's composition on r's node at r's fingerprint: in composed with r's manifest
|
||||
// in place of the controller's registered one.
|
||||
func replacedComposition(in compose.Input, r replacedController) (sdkdelivery.Composed, error) {
|
||||
shelf := make(map[string]catalogue.Manifest, len(in.Catalogue))
|
||||
for k, v := range in.Catalogue {
|
||||
shelf[k] = v
|
||||
}
|
||||
shelf[controllerModule] = r.manifest
|
||||
in.Catalogue = shelf
|
||||
all, _, err := compose.Compositions(in, r.node, map[string]string{controllerModule: r.fp}, compose.Reading)
|
||||
if err != nil {
|
||||
return sdkdelivery.Composed{}, err
|
||||
}
|
||||
key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp)
|
||||
c, ok := all[key]
|
||||
if !ok {
|
||||
return sdkdelivery.Composed{}, fmt.Errorf("%s composes no %s with the replaced build's manifest", r.node, controllerModule)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// declarationsRetiredNow says the declarations are done on the mesh now, for a raise of the bus's objects; a bus or a
|
||||
// store it cannot read is not done, so the stream is asserted as before.
|
||||
func declarationsRetiredNow(ctx context.Context, inv *inventory.Inventory) bool {
|
||||
bus, err := aBus()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer bus.Close()
|
||||
js, err := jetstream.New(bus.Conn())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
done, err := declarationsRetiredOn(ctx, inv, js)
|
||||
return err == nil && done
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// fakeBuckets is the bus's buckets by name, for deleting.
|
||||
type fakeBuckets map[string]bool
|
||||
|
||||
func (f fakeBuckets) DeleteKeyValue(_ context.Context, bucket string) error {
|
||||
if !f[bucket] {
|
||||
return jetstream.ErrBucketNotFound
|
||||
}
|
||||
delete(f, bucket)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1945/AC14 (unit half): the first start deletes the merge window's, the silent sends' and the building plans' buckets
|
||||
// and says each once; a later start finds none and says nothing; a bucket of anything else is left alone.
|
||||
func TestTheFirstStartDeletesThePlansBucketsAndSaysEachOnce(t *testing.T) {
|
||||
bus := fakeBuckets{"mesh-controller_merge-window": true, "mesh-controller_rollout-silent": true,
|
||||
"mesh-controller_plans-building": true, "mesh-controller_grants": true}
|
||||
var said []string
|
||||
deleted, err := deletePlanStateBuckets(context.Background(), bus, func(b string) { said = append(said, b) })
|
||||
if err != nil || len(deleted) != 3 || strings.Join(said, " ") != strings.Join(broker.PlanStateBuckets, " ") {
|
||||
t.Fatalf("first start: deleted %v, said %v, %v", deleted, said, err)
|
||||
}
|
||||
if !bus["mesh-controller_grants"] || len(bus) != 1 {
|
||||
t.Errorf("the bus holds %v after the first start", bus)
|
||||
}
|
||||
said = nil
|
||||
if deleted, err := deletePlanStateBuckets(context.Background(), bus, func(b string) { said = append(said, b) }); err != nil || len(deleted)+len(said) != 0 {
|
||||
t.Errorf("a later start deleted %v and said %v (%v)", deleted, said, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC6 (unit half): the controller this one replaced is composed on its node from its own build's manifest at its
|
||||
// own fingerprint: the composition holds what that manifest declares, not what the controller registered now declares.
|
||||
func TestTheReplacedControllerIsComposedFromItsOwnManifest(t *testing.T) {
|
||||
now := catalogue.Manifest{Module: controllerModule, Version: "6", Resources: []map[string]any{
|
||||
{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "six\n"}}}
|
||||
before := catalogue.Manifest{Module: controllerModule, Version: "5", Resources: []map[string]any{
|
||||
{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "five\n"}}}
|
||||
in := compose.Input{Catalogue: map[string]catalogue.Manifest{controllerModule: now},
|
||||
Nodes: []compose.Node{{Name: "anchor", Assigned: []string{controllerModule}, Stated: true}}}
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("5", 64)
|
||||
c, err := replacedComposition(in, replacedController{node: "anchor", fp: fp, manifest: before})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Module != controllerModule || c.FP != fp || c.Node != "anchor" || len(c.Resources) != 1 {
|
||||
t.Fatalf("composed %s at %s on %s with %d resources", c.Module, c.FP, c.Node, len(c.Resources))
|
||||
}
|
||||
var f struct{ Content string }
|
||||
_ = json.Unmarshal(c.Resources[0], &f)
|
||||
if f.Content != "five\n" {
|
||||
t.Errorf("the replaced controller's composition holds %q, its own manifest says five", f.Content)
|
||||
}
|
||||
if in.Catalogue[controllerModule].Version != "6" {
|
||||
t.Errorf("composing the replaced controller changed the Input's catalogue")
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC6 (unit half): the controller replaced is the newest earlier build of the controller that worked and names a
|
||||
// fingerprint of the new form, not the one registered now.
|
||||
func TestTheReplacedControllerIsItsNewestEarlierBuild(t *testing.T) {
|
||||
now, before, older := sdkdelivery.SourcePrefix2+strings.Repeat("6", 64), sdkdelivery.SourcePrefix2+strings.Repeat("5", 64),
|
||||
sdkdelivery.SourcePrefix2+strings.Repeat("4", 64)
|
||||
m := []byte(`{"module":"mesh-controller","version":"5"}`)
|
||||
builds := []inventory.Build{
|
||||
{ID: "c6", SourceFingerprint: now, Manifest: m},
|
||||
{ID: "c5-failed", SourceFingerprint: before, Failed: "no", Manifest: m},
|
||||
{ID: "c5", SourceFingerprint: before, Manifest: m},
|
||||
{ID: "c4", SourceFingerprint: older, Manifest: m},
|
||||
}
|
||||
if b, ok := replacedBuild(builds, now); !ok || b.ID != "c5" {
|
||||
t.Errorf("the replaced build is %+v (%v), want c5", b, ok)
|
||||
}
|
||||
if _, ok := replacedBuild(builds[:1], now); ok {
|
||||
t.Errorf("a controller with no earlier build replaced one")
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC14 (unit half): the declarations' stream stays while one node's have names another node-engine than the one
|
||||
// registered, or none, and goes once every node's names it.
|
||||
func TestTheDeclarationsGoOnlyOnceEveryNodeRunsTheNewEngine(t *testing.T) {
|
||||
n6 := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64)
|
||||
nodes := []string{"ace", "g14", "novox"}
|
||||
if declarationsDone(nodes, map[string]string{"ace": n6, "novox": n6}, n6) {
|
||||
t.Errorf("done while g14 says nothing (it sleeps)")
|
||||
}
|
||||
if declarationsDone(nodes, map[string]string{"ace": n6, "g14": "src2:old", "novox": n6}, n6) {
|
||||
t.Errorf("done while g14 runs the old node-engine")
|
||||
}
|
||||
if !declarationsDone(nodes, map[string]string{"ace": n6, "g14": n6, "novox": n6}, n6) {
|
||||
t.Errorf("not done with every node on the new node-engine")
|
||||
}
|
||||
if declarationsDone(nodes, map[string]string{"ace": "", "g14": "", "novox": ""}, "") || declarationsDone(nil, nil, n6) {
|
||||
t.Errorf("done with no fingerprint registered, or no node")
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// composeTrace, when set, is told what composing a node saw and decided: the Input's credential state before it composes, and for
|
||||
// each module the digest it composed against the digest already written. A test sets it to read why a composition did or did not
|
||||
// move; the controller never does.
|
||||
var composeTrace func(string)
|
||||
|
||||
// traceInput says what the node's composition will read of the pair credentials: those no row holds (would-make) and those held.
|
||||
func traceInput(ctx context.Context, open *stores, node string) {
|
||||
in, err := inputFor(ctx, open)
|
||||
if err != nil {
|
||||
composeTrace(fmt.Sprintf("%s: the Input cannot be read: %v", node, err))
|
||||
return
|
||||
}
|
||||
composeTrace(fmt.Sprintf("%s: before composing, would make %+v; credentials held %+v", node, in.wouldMake, in.Credentials))
|
||||
}
|
||||
|
||||
// traceComposed says, per composition of a node, the digest composed now against the one written.
|
||||
func traceComposed(ctx context.Context, at composedPlace, node string, comps map[string]sdkdelivery.Composed) {
|
||||
keys := make([]string, 0, len(comps))
|
||||
for k := range comps {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
now, err := sdkdelivery.ComposedDigestOf(comps[key])
|
||||
if err != nil {
|
||||
composeTrace(fmt.Sprintf("%s: %s: no digest: %v", node, key, err))
|
||||
continue
|
||||
}
|
||||
written := "none"
|
||||
if e, err := at.digests.Get(ctx, key); err == nil {
|
||||
if v, err := sdkdelivery.DecodeComposedDigest(e.Value()); err == nil {
|
||||
written = v.Digest
|
||||
}
|
||||
}
|
||||
same := "CHANGED"
|
||||
if written == now {
|
||||
same = "same"
|
||||
}
|
||||
composeTrace(fmt.Sprintf("%s: %s: composed %.19s, written %.19s: %s", node, key, now, written, same))
|
||||
}
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// natsComposedWith is the bus module's composition on novox with the given user list: the catalogue's nats manifest as
|
||||
// captured (testdata/beside/mesh-catalog/modules/nats/module.json.captured), its archives, which only a registration
|
||||
// turns into references, left out, and its container's built image given as a reference.
|
||||
func natsComposedWith(t *testing.T, users string) sdkdelivery.Composed {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../testdata/beside/mesh-catalog/modules/nats/module.json.captured")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m catalogue.Manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var kept []map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if _, templated := r["template"]; templated {
|
||||
continue // a template is rendered from the build's templates archive, which only a registration fetches
|
||||
}
|
||||
switch r["type"] {
|
||||
case "archive":
|
||||
continue
|
||||
case "container":
|
||||
// Its image is named by a build's artifact, which only a registration records: given here as a reference.
|
||||
delete(r, "artifact")
|
||||
r["image"] = "nats@sha256:" + strings.Repeat("1", 64)
|
||||
}
|
||||
kept = append(kept, r)
|
||||
}
|
||||
m.Resources = kept
|
||||
key := base64.StdEncoding.EncodeToString(make([]byte, 32))
|
||||
in := compose.Input{Catalogue: map[string]catalogue.Manifest{m.Module: m}, BusUsers: users,
|
||||
OwnSecrets: []compose.OwnSecretRow{{Node: "novox", Module: m.Module, Name: "broker", Origin: "made", SealedTo: key,
|
||||
Sealed: "SEALED-BY-THE-TEST"}},
|
||||
Nodes: []compose.Node{{Name: "novox", Assigned: []string{m.Module}, Stated: true, SealingKey: key,
|
||||
Capabilities: map[string]bool{"container-runtime": true, "systemd": true}}}}
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||
all, _, err := compose.Compositions(in, "novox", map[string]string{m.Module: fp}, compose.Foreseeing)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := sdkdelivery.ComposedKey("novox", m.Module, fp)
|
||||
c, ok := all[k]
|
||||
if !ok {
|
||||
t.Fatalf("no composition of %s: %v", m.Module, all)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// 1945/AC12 and the review's A2 (ADR 0539 clause 7): a revision of the bus module's composition that changes only its
|
||||
// user list, a file in a directory its container mounts, is judged neither and said to change only the user list, so
|
||||
// a new credential never waits for a person's yes; a revision that changes anything else beside it is judged as any.
|
||||
func TestARevisionOfOnlyTheUserListMovesNothing(t *testing.T) {
|
||||
at, _, _, haves := aPlace(t, "novox")
|
||||
ctx := context.Background()
|
||||
before := natsComposedWith(t, "authorization { users = [ {user: a} ] }\n")
|
||||
after := natsComposedWith(t, "authorization { users = [ {user: a}, {user: b} ] }\n")
|
||||
if d1, _ := sdkdelivery.ComposedDigestOf(before); d1 == func() string { d, _ := sdkdelivery.ComposedDigestOf(after); return d }() {
|
||||
t.Fatal("the two user lists composed the same")
|
||||
}
|
||||
key, _ := sdkdelivery.ComposedKey("novox", before.Module, before.FP)
|
||||
if _, err := writeComposed(ctx, at, key, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runs, _ := sdkdelivery.ComposedDigestOf(before)
|
||||
haves[before.Module] = sdkdelivery.Have{FP: before.FP, Composed: runs, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
digest, _ := sdkdelivery.ComposedDigestOf(after)
|
||||
v, err := judgeMove(ctx, at, key, after, digest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.Judgement != sdkdelivery.JudgedNeither || !v.UserListOnly || v.Against != runs {
|
||||
t.Errorf("a user-list-only revision is judged %+v", v)
|
||||
}
|
||||
// The user list and a container's image beside it: judged as any move, and not user-list-only.
|
||||
other := after
|
||||
other.Resources = append([]json.RawMessage{}, after.Resources...)
|
||||
for i, r := range other.Resources {
|
||||
var head map[string]any
|
||||
_ = json.Unmarshal(r, &head)
|
||||
if head["type"] == "container" {
|
||||
head["image"] = "nats@sha256:" + strings.Repeat("9", 64)
|
||||
other.Resources[i], _ = json.Marshal(head)
|
||||
}
|
||||
}
|
||||
d2, _ := sdkdelivery.ComposedDigestOf(other)
|
||||
v, err = judgeMove(ctx, at, key, other, d2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v.UserListOnly || v.Judgement != sdkdelivery.JudgedRecreates {
|
||||
t.Errorf("a new image beside the user list is judged %+v", v)
|
||||
}
|
||||
}
|
||||
@@ -1,370 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The compositions (novox/hq ADR 0539 clauses 1 to 5). Every act that sent a declaration writes compositions instead:
|
||||
// the controller composes a node as it composed a send (composeSend: the same composition, the same makes), splits the
|
||||
// declaration by module into one composition per module at the fingerprint the module is registered at, and the node's
|
||||
// own resources, signs each with its key, and writes those that differ, by compare-and-set while it holds the lease:
|
||||
// the composition into the node's bucket `composed-<node>`, which only that node reads, and its digest with the
|
||||
// judgement of the move (composed_judge.go) into `mesh-controller_composed`, which the assigner reads. A file over 64
|
||||
// KiB goes by digest into the node's object bucket first. Nothing is published on a node's declaration subject.
|
||||
//
|
||||
// **A module with no fingerprint of the new form goes with the node's own composition.** The mesh composes resources
|
||||
// of modules it provides itself, built by nobody (the private network's mesh-wireguard), and a module registered
|
||||
// before fingerprints of the new form has none either: no want can ever name such a module (the assigner's first rule
|
||||
// asks for its done build), so its resources ride in `<node>.node`, which the node's own want names, their `module`
|
||||
// taken off (a composition holds only its own module's resources); their ids still name the module.
|
||||
//
|
||||
// **Derived state, pruned** (clause 4): a module's compositions on a node are kept for its five newest fingerprints and
|
||||
// for any a want or the node's have names; the rest are deleted with their digests.
|
||||
|
||||
// composedBucketsKV is what the writer needs of a key-value bucket: jetstream.KeyValue, or a test's.
|
||||
type composedBucketsKV interface {
|
||||
Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error)
|
||||
Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error)
|
||||
Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error)
|
||||
Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error
|
||||
ListKeys(ctx context.Context, opts ...jetstream.WatchOpt) (jetstream.KeyLister, error)
|
||||
}
|
||||
|
||||
// composedPlace is where the writer writes a node's compositions and reads what decides them: the node's bucket, the
|
||||
// digests, the node's object bucket, the wants and the haves.
|
||||
type composedPlace struct {
|
||||
node string
|
||||
bucket composedBucketsKV
|
||||
digests composedBucketsKV
|
||||
objects objectStore
|
||||
// wanted is the fingerprint the node's want names for a module, "" when none; ran the composition digest its have
|
||||
// names (the move's judgement reads it) and the fingerprint it runs.
|
||||
wanted func(ctx context.Context, module string) (string, error)
|
||||
ran func(ctx context.Context, module string) (sdkdelivery.Have, bool, error)
|
||||
// lastApplied is the module's slice of the declaration the node last applied, from the kept stream of declarations
|
||||
// (ADR 0540 clause 9), and that declaration's digest; false when there is none to read. Nil judges without it.
|
||||
lastApplied func(ctx context.Context, module string) (slice []json.RawMessage, digest string, found bool, err error)
|
||||
// policy is a module's upgrade policy, where it comes from and why (catalogue.UpgradeOf), carried beside the
|
||||
// judgement for the assigner's hold (ADR 0539 clause 14).
|
||||
policy func(module string) (policy, from, why string)
|
||||
// sign signs a composition with the controller's key.
|
||||
sign func(c sdkdelivery.Composed) (sdkdelivery.Composed, error)
|
||||
}
|
||||
|
||||
// compositionsOf splits a node's declaration body, as a send composed it, into its compositions: each module's at the
|
||||
// fingerprint fps gives it, the node's own with its adoption and provisioner, and every resource of a module fps names
|
||||
// no fingerprint for in the node's own, its module taken off it and named in moved. Data says the module's manifest
|
||||
// declares data.
|
||||
func compositionsOf(node string, body []byte, fps map[string]string, manifests map[string]catalogue.Manifest, runs []string) (
|
||||
out map[string]sdkdelivery.Composed, moved []string, err error) {
|
||||
|
||||
var envelope struct {
|
||||
Resources []map[string]json.RawMessage `json:"resources"`
|
||||
Adoption *sdkdecl.Adoption `json:"adoption"`
|
||||
Filled []string `json:"filled_files"`
|
||||
Provisioner string `json:"provisioner_account"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return nil, nil, fmt.Errorf("%s's declaration: %w", node, err)
|
||||
}
|
||||
inModule := map[string]bool{}
|
||||
resources := make([]json.RawMessage, 0, len(envelope.Resources))
|
||||
for _, r := range envelope.Resources {
|
||||
var module string
|
||||
if raw, ok := r["module"]; ok {
|
||||
_ = json.Unmarshal(raw, &module)
|
||||
}
|
||||
if module != "" && !strings.HasPrefix(fps[module], sdkdelivery.SourcePrefix2) {
|
||||
delete(r, "module")
|
||||
if !inModule[module] {
|
||||
inModule[module] = true
|
||||
moved = append(moved, module)
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
resources = append(resources, raw)
|
||||
}
|
||||
// A filled file of a module moved into the node's own composition is no module's file any more: the provisioner
|
||||
// fills it all the same, but a composition carries only its own module's filled paths.
|
||||
var filled []string
|
||||
for _, p := range envelope.Filled {
|
||||
if !filledByMoved(envelope.Resources, p, inModule) {
|
||||
filled = append(filled, p)
|
||||
}
|
||||
}
|
||||
sort.Strings(moved)
|
||||
out, err = sdkdelivery.SplitComposed(node, resources, fps, filled,
|
||||
sdkdelivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// A module the node runs that composes no resource of its own (all its needs met by others) still has its
|
||||
// composition, empty, at its fingerprint: its want names that composition (the assigner's seventh rule), and without
|
||||
// one no want could ever be written.
|
||||
for _, module := range runs {
|
||||
fp := fps[module]
|
||||
if !strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
|
||||
continue
|
||||
}
|
||||
key, err := sdkdelivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := out[key]; !ok {
|
||||
out[key] = sdkdelivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{}}
|
||||
}
|
||||
}
|
||||
for key, c := range out {
|
||||
if m, ok := manifests[c.Module]; ok && c.Module != "" {
|
||||
c.Data = len(m.DataItems()) > 0
|
||||
out[key] = c
|
||||
}
|
||||
}
|
||||
return out, moved, nil
|
||||
}
|
||||
|
||||
// filledByMoved says a filled path is a file of a module whose resources were moved into the node's own composition.
|
||||
func filledByMoved(resources []map[string]json.RawMessage, path string, moved map[string]bool) bool {
|
||||
for _, r := range resources {
|
||||
var kind, at, id string
|
||||
_ = json.Unmarshal(r["type"], &kind)
|
||||
_ = json.Unmarshal(r["path"], &at)
|
||||
_ = json.Unmarshal(r["id"], &id)
|
||||
if kind == string(sdkdecl.TypeFile) && at == path {
|
||||
if module, ok := sdkdecl.ModuleGuess(id); ok && moved[module] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// writeComposed writes one composition and its digest where they differ from what the buckets hold: the files by
|
||||
// digest first, then the composition, then its digest and judgement, so a reader that finds the digest finds the
|
||||
// composition and its files. It answers whether it wrote anything.
|
||||
func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdelivery.Composed) (bool, error) {
|
||||
c, objects, err := c.ByDigest()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(objects) > 0 {
|
||||
if at.objects == nil {
|
||||
return false, fmt.Errorf("%s holds a file over 64 KiB and %s's object bucket is not open", key, at.node)
|
||||
}
|
||||
named := make(map[string][]byte, len(objects))
|
||||
for digest, body := range objects {
|
||||
name, err := sdkdecl.ObjectName(digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
named[name] = body
|
||||
}
|
||||
if err := putObjects(ctx, at.objects, named); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
digest, err := sdkdelivery.ComposedDigestOf(c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
wrote := false
|
||||
held, revision, err := readComposed(ctx, at.bucket, key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if heldDigest, _ := sdkdelivery.ComposedDigestOf(held); revision == 0 || heldDigest != digest {
|
||||
signed, err := at.sign(c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
value, err := sdkdelivery.EncodeComposed(signed)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := casWrite(ctx, at.bucket, key, value, revision); err != nil {
|
||||
return false, err
|
||||
}
|
||||
wrote = true
|
||||
}
|
||||
judged, err := judgeMove(ctx, at, key, c, digest)
|
||||
if composeTrace != nil {
|
||||
composeTrace(fmt.Sprintf("%s: %s: written %v, judged %s %q user-list-only %v against %.19s, err %v", at.node, key, wrote,
|
||||
judged.Judgement, judged.Words, judged.UserListOnly, judged.Against, err))
|
||||
}
|
||||
if err != nil {
|
||||
return wrote, err
|
||||
}
|
||||
written, err := keepDigest(ctx, at.digests, key, judged)
|
||||
return wrote || written, err
|
||||
}
|
||||
|
||||
// readComposed is the composition a bucket holds under key and its revision; revision 0 when there is none.
|
||||
func readComposed(ctx context.Context, kv composedBucketsKV, key string) (sdkdelivery.Composed, uint64, error) {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return sdkdelivery.Composed{}, 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return sdkdelivery.Composed{}, 0, err
|
||||
}
|
||||
c, err := sdkdelivery.DecodeComposed(e.Value())
|
||||
if err != nil {
|
||||
// A value this controller cannot read is written over: it is derived, and composed again here.
|
||||
return sdkdelivery.Composed{}, e.Revision(), nil
|
||||
}
|
||||
return c, e.Revision(), nil
|
||||
}
|
||||
|
||||
// keepDigest writes the digest and judgement beside a composition when it differs from what the bucket holds.
|
||||
func keepDigest(ctx context.Context, kv composedBucketsKV, key string, v sdkdelivery.ComposedDigest) (bool, error) {
|
||||
value, err := sdkdelivery.EncodeComposedDigest(v)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
v.V = sdkdelivery.Version
|
||||
var revision uint64
|
||||
e, err := kv.Get(ctx, key)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
case err != nil:
|
||||
return false, err
|
||||
default:
|
||||
revision = e.Revision()
|
||||
if held, derr := sdkdelivery.DecodeComposedDigest(e.Value()); derr == nil && sdkdelivery.SameComposedDigest(held, v) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, casWrite(ctx, kv, key, value, revision)
|
||||
}
|
||||
|
||||
// casWrite creates key when revision is 0 and otherwise replaces the revision read: a second writer that wrote in
|
||||
// between makes this one fail, said, and the next event composes again.
|
||||
func casWrite(ctx context.Context, kv composedBucketsKV, key string, value []byte, revision uint64) error {
|
||||
var err error
|
||||
if revision == 0 {
|
||||
_, err = kv.Create(ctx, key, value)
|
||||
} else {
|
||||
_, err = kv.Update(ctx, key, value, revision)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeNodeCompositions writes every composition of a node that differs, and prunes what is no longer kept. It answers
|
||||
// the keys it wrote.
|
||||
func writeNodeCompositions(ctx context.Context, at composedPlace, comps map[string]sdkdelivery.Composed) ([]string, error) {
|
||||
keys := make([]string, 0, len(comps))
|
||||
for k := range comps {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var wrote []string
|
||||
var errs []error
|
||||
for _, key := range keys {
|
||||
w, err := writeComposed(ctx, at, key, comps[key])
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if w {
|
||||
wrote = append(wrote, key)
|
||||
}
|
||||
}
|
||||
if err := pruneCompositions(ctx, at); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return wrote, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// pruneCompositions deletes, for each module of the node, the compositions outside its ComposedKeep newest fingerprints
|
||||
// that neither its want nor its have names, each with its digest (ADR 0539 clause 4). Newest is by when each was last
|
||||
// written: the controller writes only the fingerprint a module is registered at, so an older one is never written again.
|
||||
func pruneCompositions(ctx context.Context, at composedPlace) error {
|
||||
lister, err := at.bucket.ListKeys(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
type kept struct {
|
||||
key, fp string
|
||||
at time.Time
|
||||
}
|
||||
byModule := map[string][]kept{}
|
||||
for key := range lister.Keys() {
|
||||
node, module, fp, isNode, err := sdkdelivery.ParseComposedKey(key)
|
||||
if err != nil || isNode || node != at.node {
|
||||
continue
|
||||
}
|
||||
e, err := at.bucket.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
byModule[module] = append(byModule[module], kept{key, fp, e.Created()})
|
||||
}
|
||||
var errs []error
|
||||
for module, ks := range byModule {
|
||||
if len(ks) <= sdkdelivery.ComposedKeep {
|
||||
continue
|
||||
}
|
||||
sort.Slice(ks, func(i, j int) bool { return ks[i].at.After(ks[j].at) })
|
||||
wanted, err := at.wanted(ctx, module)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
have, _, err := at.ran(ctx, module)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
for _, k := range ks[sdkdelivery.ComposedKeep:] {
|
||||
if k.fp == wanted || k.fp == have.FP {
|
||||
continue
|
||||
}
|
||||
if err := at.bucket.Delete(ctx, k.key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if err := at.digests.Delete(ctx, k.key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// controllerSigner signs a composition with the controller's active key, which its identity store keeps and never hands
|
||||
// out (as it signed declarations).
|
||||
func controllerSigner(ctx context.Context, open *stores) (func(sdkdelivery.Composed) (sdkdelivery.Composed, error), error) {
|
||||
ident, err := open.Identity(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := ident.Active(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
public := ed25519.PublicKey(key.Public)
|
||||
return func(c sdkdelivery.Composed) (sdkdelivery.Composed, error) {
|
||||
return sdkdelivery.SignComposedBy(c, public, func(m []byte) ([]byte, error) { return ident.Sign(ctx, m) })
|
||||
}, nil
|
||||
}
|
||||
@@ -1,362 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// fakeComposedKV is a key-value bucket in memory: revisions, a creation time per write, history, compare-and-set.
|
||||
type fakeComposedKV struct {
|
||||
name string
|
||||
rev uint64
|
||||
clock time.Time
|
||||
history map[string][]fakeComposedEntry
|
||||
writes int
|
||||
deletes int
|
||||
}
|
||||
|
||||
type fakeComposedEntry struct {
|
||||
bucket, key string
|
||||
value []byte
|
||||
rev uint64
|
||||
at time.Time
|
||||
op jetstream.KeyValueOp
|
||||
}
|
||||
|
||||
func (e fakeComposedEntry) Bucket() string { return e.bucket }
|
||||
func (e fakeComposedEntry) Key() string { return e.key }
|
||||
func (e fakeComposedEntry) Value() []byte { return e.value }
|
||||
func (e fakeComposedEntry) Revision() uint64 { return e.rev }
|
||||
func (e fakeComposedEntry) Created() time.Time { return e.at }
|
||||
func (e fakeComposedEntry) Delta() uint64 { return 0 }
|
||||
func (e fakeComposedEntry) Operation() jetstream.KeyValueOp { return e.op }
|
||||
|
||||
func newFakeComposedKV(name string) *fakeComposedKV {
|
||||
return &fakeComposedKV{name: name, clock: time.Date(2026, 10, 24, 17, 0, 0, 0, time.UTC), history: map[string][]fakeComposedEntry{}}
|
||||
}
|
||||
|
||||
func (f *fakeComposedKV) last(key string) (fakeComposedEntry, bool) {
|
||||
h := f.history[key]
|
||||
if len(h) == 0 || h[len(h)-1].op != jetstream.KeyValuePut {
|
||||
return fakeComposedEntry{}, false
|
||||
}
|
||||
return h[len(h)-1], true
|
||||
}
|
||||
|
||||
func (f *fakeComposedKV) put(key string, value []byte, op jetstream.KeyValueOp) uint64 {
|
||||
f.rev++
|
||||
f.clock = f.clock.Add(time.Second)
|
||||
f.history[key] = append(f.history[key], fakeComposedEntry{f.name, key, value, f.rev, f.clock, op})
|
||||
return f.rev
|
||||
}
|
||||
|
||||
func (f *fakeComposedKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
e, ok := f.last(key)
|
||||
if !ok {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func (f *fakeComposedKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
||||
if _, ok := f.last(key); ok {
|
||||
return 0, jetstream.ErrKeyExists
|
||||
}
|
||||
f.writes++
|
||||
return f.put(key, value, jetstream.KeyValuePut), nil
|
||||
}
|
||||
|
||||
func (f *fakeComposedKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
e, ok := f.last(key)
|
||||
if !ok || e.rev != revision {
|
||||
return 0, fmt.Errorf("wrong last sequence")
|
||||
}
|
||||
f.writes++
|
||||
return f.put(key, value, jetstream.KeyValuePut), nil
|
||||
}
|
||||
|
||||
func (f *fakeComposedKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
if _, ok := f.last(key); !ok {
|
||||
return jetstream.ErrKeyNotFound
|
||||
}
|
||||
f.deletes++
|
||||
f.put(key, nil, jetstream.KeyValueDelete)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeComposedKV) History(_ context.Context, key string, _ ...jetstream.WatchOpt) ([]jetstream.KeyValueEntry, error) {
|
||||
h := f.history[key]
|
||||
if len(h) == 0 {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
out := make([]jetstream.KeyValueEntry, len(h))
|
||||
for i, e := range h {
|
||||
out[i] = e
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type fakeLister struct{ keys chan string }
|
||||
|
||||
func (l fakeLister) Keys() <-chan string { return l.keys }
|
||||
func (l fakeLister) Stop() error { return nil }
|
||||
|
||||
func (f *fakeComposedKV) ListKeys(_ context.Context, _ ...jetstream.WatchOpt) (jetstream.KeyLister, error) {
|
||||
var keys []string
|
||||
for k := range f.history {
|
||||
if _, ok := f.last(k); ok {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
ch := make(chan string, len(keys))
|
||||
for _, k := range keys {
|
||||
ch <- k
|
||||
}
|
||||
close(ch)
|
||||
return fakeLister{ch}, nil
|
||||
}
|
||||
|
||||
// fakeObjects is a node's object bucket in memory.
|
||||
type fakeObjects map[string][]byte
|
||||
|
||||
func (o fakeObjects) Has(_ context.Context, name string) (bool, bool, error) {
|
||||
_, ok := o[name]
|
||||
return ok, false, nil
|
||||
}
|
||||
func (o fakeObjects) Put(_ context.Context, name string, content []byte) error {
|
||||
o[name] = content
|
||||
return nil
|
||||
}
|
||||
|
||||
// aPlace is a node's buckets in memory, signed with a key of the test's, the want and the have given.
|
||||
func aPlace(t *testing.T, node string) (composedPlace, ed25519.PublicKey, map[string]string, map[string]sdkdelivery.Have) {
|
||||
t.Helper()
|
||||
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wants, haves := map[string]string{}, map[string]sdkdelivery.Have{}
|
||||
at := composedPlace{node: node, bucket: newFakeComposedKV("composed-" + node), digests: newFakeComposedKV("mesh-controller_composed"),
|
||||
objects: fakeObjects{},
|
||||
wanted: func(_ context.Context, module string) (string, error) { return wants[module], nil },
|
||||
ran: func(_ context.Context, module string) (sdkdelivery.Have, bool, error) {
|
||||
h, ok := haves[module]
|
||||
return h, ok, nil
|
||||
},
|
||||
policy: func(string) (string, string, string) { return catalogue.PolicyRoll, catalogue.FromDefault, "" },
|
||||
sign: func(c sdkdelivery.Composed) (sdkdelivery.Composed, error) { return sdkdelivery.SignComposed(c, priv) },
|
||||
}
|
||||
return at, pub, wants, haves
|
||||
}
|
||||
|
||||
// capturedBody is a node's declaration as the controller's `plan` verb answered it on 2026-10-24 at 16:19, the body a
|
||||
// send of declarationFor carries (testdata/declaration-<node>.json.gz, copied from mesh-sdk's; the resources carrying
|
||||
// a sealed value or a secret taken out), and a fingerprint standing in for each module's build.
|
||||
func capturedBody(t *testing.T, node string) ([]byte, map[string]string) {
|
||||
t.Helper()
|
||||
f, err := os.Open("testdata/declaration-" + node + ".json.gz")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
z, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := io.ReadAll(z)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var d struct {
|
||||
Builds map[string]string `json:"builds"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fps := map[string]string{}
|
||||
for m, b := range d.Builds {
|
||||
fps[m] = sdkdelivery.SourcePrefix2 + fmt.Sprintf("%064x", []byte(b))[:64]
|
||||
}
|
||||
return raw, fps
|
||||
}
|
||||
|
||||
func resourceIDs(t *testing.T, raws []json.RawMessage) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
for _, r := range raws {
|
||||
var head struct{ ID string }
|
||||
if err := json.Unmarshal(r, &head); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, head.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// 1945/AC1: for a captured node's declaration, the union of its module compositions and its own is the declaration's
|
||||
// resources, each once; each module's at its fingerprint; a module with no fingerprint of the new form (the private
|
||||
// network's mesh-wireguard, which nobody builds) rides in the node's own.
|
||||
func TestTheCompositionsOfANodeAreItsDeclaration(t *testing.T) {
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
body, fps := capturedBody(t, node)
|
||||
var d struct {
|
||||
Resources []json.RawMessage `json:"resources"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &d)
|
||||
comps, moved, err := compositionsOf(node, body, fps, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
if strings.Join(moved, ",") != "mesh-wireguard" {
|
||||
t.Errorf("%s: moved into the node's own %v, want mesh-wireguard", node, moved)
|
||||
}
|
||||
var union []string
|
||||
for key, c := range comps {
|
||||
_, module, fp, isNode, err := sdkdelivery.ParseComposedKey(key)
|
||||
if err != nil || (!isNode && fp != fps[module]) {
|
||||
t.Errorf("%s: %s is at %s (%v)", node, key, fp, err)
|
||||
}
|
||||
if _, err := sdkdelivery.EncodeComposed(c); err != nil {
|
||||
t.Errorf("%s: %s: %v", node, key, err)
|
||||
}
|
||||
union = append(union, resourceIDs(t, c.Resources)...)
|
||||
}
|
||||
want := resourceIDs(t, d.Resources)
|
||||
sort.Strings(union)
|
||||
sort.Strings(want)
|
||||
if strings.Join(union, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("%s: the compositions hold %d resources, the declaration %d, or they differ", node, len(union), len(want))
|
||||
}
|
||||
own, _ := sdkdelivery.ComposedNodeKey(node)
|
||||
if ids := strings.Join(resourceIDs(t, comps[own].Resources), " "); !strings.Contains(ids, "mesh-wireguard.") {
|
||||
t.Errorf("%s: the node's own composition does not hold mesh-wireguard's resources: %s", node, ids)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC1: a composition is written signed by the controller's key with its digest beside it; written again
|
||||
// unchanged it writes nothing; a changed input writes a new revision of that key alone, by compare-and-set.
|
||||
func TestACompositionIsWrittenOnceAndAgainOnlyWhenItChanges(t *testing.T) {
|
||||
body, fps := capturedBody(t, "g14")
|
||||
comps, _, err := compositionsOf("g14", body, fps, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at, pub, _, _ := aPlace(t, "g14")
|
||||
wrote, err := writeNodeCompositions(context.Background(), at, comps)
|
||||
if err != nil || len(wrote) != len(comps) {
|
||||
t.Fatalf("first write: %d of %d (%v)", len(wrote), len(comps), err)
|
||||
}
|
||||
for key := range comps {
|
||||
e, err := at.bucket.Get(context.Background(), key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, err := sdkdelivery.DecodeComposed(e.Value())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest, err := sdkdelivery.VerifyComposed(c, []ed25519.PublicKey{pub})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", key, err)
|
||||
}
|
||||
d, err := at.digests.Get(context.Background(), key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := sdkdelivery.DecodeComposedDigest(d.Value())
|
||||
if err != nil || v.Digest != digest {
|
||||
t.Errorf("%s: the digest beside it is %q, the composition's %s (%v)", key, v.Digest, digest, err)
|
||||
}
|
||||
}
|
||||
writes := at.bucket.(*fakeComposedKV).writes + at.digests.(*fakeComposedKV).writes
|
||||
if again, err := writeNodeCompositions(context.Background(), at, comps); err != nil || len(again) != 0 {
|
||||
t.Errorf("written again unchanged: %v (%v)", again, err)
|
||||
}
|
||||
if now := at.bucket.(*fakeComposedKV).writes + at.digests.(*fakeComposedKV).writes; now != writes {
|
||||
t.Errorf("an unchanged write wrote %d values", now-writes)
|
||||
}
|
||||
|
||||
key, _ := sdkdelivery.ComposedKey("g14", "fail2ban", fps["fail2ban"])
|
||||
changed := comps[key]
|
||||
changed.Data = true
|
||||
comps[key] = changed
|
||||
wrote, err = writeNodeCompositions(context.Background(), at, comps)
|
||||
if err != nil || len(wrote) != 1 || wrote[0] != key {
|
||||
t.Errorf("a changed composition wrote %v (%v), want %s alone", wrote, err, key)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC1: a module's compositions outside its five newest fingerprints are deleted with their digests, unless its
|
||||
// want or its have names one.
|
||||
func TestCompositionsOutsideTheFiveNewestAreDeleted(t *testing.T) {
|
||||
at, _, wants, haves := aPlace(t, "ace")
|
||||
file := json.RawMessage(`{"id":"m.f","type":"file","module":"m","path":"/etc/m","content":"x"}`)
|
||||
fp := func(i int) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", i), 64) }
|
||||
for i := 1; i <= 8; i++ {
|
||||
c := sdkdelivery.Composed{Node: "ace", Module: "m", FP: fp(i), Resources: []json.RawMessage{file}}
|
||||
key, _ := sdkdelivery.ComposedKey("ace", "m", fp(i))
|
||||
if _, err := writeNodeCompositions(context.Background(), at, map[string]sdkdelivery.Composed{key: c}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i == 1 {
|
||||
wants["m"] = fp(1)
|
||||
}
|
||||
if i == 2 {
|
||||
haves["m"] = sdkdelivery.Have{FP: fp(2), Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
|
||||
}
|
||||
}
|
||||
if err := pruneCompositions(context.Background(), at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 1; i <= 8; i++ {
|
||||
key, _ := sdkdelivery.ComposedKey("ace", "m", fp(i))
|
||||
_, err := at.bucket.Get(context.Background(), key)
|
||||
_, derr := at.digests.Get(context.Background(), key)
|
||||
kept := err == nil && derr == nil
|
||||
wantKept := i >= 4 || i == 1 || i == 2
|
||||
if kept != wantKept {
|
||||
t.Errorf("fingerprint %d kept %v, want %v (the wanted 1, the run 2, the five newest 4 to 8)", i, kept, wantKept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945 (found by the served test): a module the node runs that composes no resource of its own still has its
|
||||
// composition, empty, at its fingerprint, so a want can name it; a module the node runs with no fingerprint of the new
|
||||
// form gets none.
|
||||
func TestAModuleWithNoResourceOfItsOwnStillHasItsComposition(t *testing.T) {
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
comps, _, err := compositionsOf("laptop", body, map[string]string{"board": fp}, nil, []string{"board", "legacy"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := sdkdelivery.ComposedKey("laptop", "board", fp)
|
||||
c, ok := comps[key]
|
||||
if !ok || len(c.Resources) != 0 {
|
||||
t.Fatalf("the board's composition: %+v (%v)", c, ok)
|
||||
}
|
||||
if _, err := sdkdelivery.EncodeComposed(c); err != nil {
|
||||
t.Errorf("an empty composition does not encode: %v", err)
|
||||
}
|
||||
for k := range comps {
|
||||
if strings.Contains(k, "legacy") {
|
||||
t.Errorf("a module with no fingerprint of the new form was composed: %s", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// stalledConditions are n open conditions like those of 2026-10-08: one per delivery held past its bound,
|
||||
// each keeping all its evidence.
|
||||
func stalledConditions(n int) []conditions.Condition {
|
||||
at := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC)
|
||||
var out []conditions.Condition
|
||||
for i := 0; i < n; i++ {
|
||||
id := fmt.Sprintf("d-%012x", 0x51a11ed+i)
|
||||
lines := stalledObservations([]stalledLine{{ID: id, State: "checking", For: "13h2m", Bound: "45m",
|
||||
H2: "close: recheck", Says: "the check was asked and nothing has answered"}})
|
||||
o := lines[0]
|
||||
c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Subject: conditions.Subject{Scope: o.Scope, ID: o.ID},
|
||||
Severity: o.Severity, Summary: o.Summary, Source: probeDeliveriesID, Raised: at, LastObserved: at,
|
||||
Observations: 160, Count: 1, Resolver: conditions.ResolverSelf}
|
||||
for e := 0; e < conditions.KeptEvidence; e++ {
|
||||
c.Evidence = append(c.Evidence, conditions.Evidence{At: at.Add(time.Duration(e) * 5 * time.Minute),
|
||||
Said: "checking for " + (13*time.Hour + time.Duration(e)*time.Minute).String()})
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 314: a list of hundreds of open conditions carries each one's newest evidence, with
|
||||
// how much more it keeps and where to read it, and the verb's answer says it once — as data, not again
|
||||
// as the text the command printed. On 2026-10-08, 689 stalled deliveries made that answer more than the
|
||||
// bus carries in one message; now it is less than half of one.
|
||||
func TestAListOfConditionsCarriesEachInBrief(t *testing.T) {
|
||||
list := stalledConditions(689)
|
||||
printed, _ := json.MarshalIndent(map[string]any{"conditions": inBrief(list), "open": len(list),
|
||||
"counted": counted(list)}, "", " ")
|
||||
var parsed any
|
||||
if err := json.Unmarshal(printed, &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wire, _ := json.Marshal(map[string]any{"result": verbAnswer{OK: true, Answer: parsed,
|
||||
Output: "its answer, as data, is `answer`\n"}})
|
||||
before, _ := json.MarshalIndent(map[string]any{"conditions": list}, "", " ")
|
||||
t.Logf("689 stalled deliveries: %d bytes on the wire, where the whole list printed was %d and was sent twice",
|
||||
len(wire), len(before))
|
||||
if len(wire) >= 512<<10 {
|
||||
t.Fatalf("the answer of 689 conditions is %d bytes, not under half of one message of the bus", len(wire))
|
||||
}
|
||||
var doc struct {
|
||||
Conditions []conditionInBrief `json:"conditions"`
|
||||
Counted map[string]int `json:"counted"`
|
||||
}
|
||||
if err := json.Unmarshal(printed, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := doc.Conditions[0]
|
||||
if len(doc.Conditions) != 689 || len(c.Evidence) != 1 || c.EvidenceKept != conditions.KeptEvidence ||
|
||||
c.Key != list[0].Key || c.Summary != list[0].Summary || !strings.Contains(c.More, "key="+c.Key) {
|
||||
t.Fatalf("a condition in brief reads %+v", c)
|
||||
}
|
||||
if doc.Counted["D14 stalled"] != 689 {
|
||||
t.Fatalf("counted %v", doc.Counted)
|
||||
}
|
||||
if one := inBrief(list[:1])[0]; len(list[0].Evidence) != conditions.KeptEvidence || one.Evidence[0] != list[0].Evidence[0] {
|
||||
t.Fatal("the brief changed the condition it was made from, or kept other than the newest")
|
||||
}
|
||||
}
|
||||
|
||||
// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and
|
||||
// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314).
|
||||
@@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -15,6 +18,22 @@ const n8nWithAGroup = `{"module":"n8n","version":"1",
|
||||
"image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0",
|
||||
"groups":["${config:media-gid}"]}]}`
|
||||
|
||||
// printedBy runs reportUnhostable and returns what it printed.
|
||||
func printedBy(t *testing.T, node string, plan catalogue.Resolution) string {
|
||||
t.Helper()
|
||||
stdout := os.Stdout
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Stdout = w
|
||||
reportUnhostable(node, plan)
|
||||
w.Close()
|
||||
os.Stdout = stdout
|
||||
out, _ := io.ReadAll(r)
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named.
|
||||
func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) {
|
||||
m, err := catalogue.ParseManifest([]byte(n8nWithAGroup))
|
||||
@@ -29,6 +48,9 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing
|
||||
if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 {
|
||||
t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable)
|
||||
}
|
||||
if out := printedBy(t, "ace", got); out != "" {
|
||||
t.Errorf("a node that reads groups was told: %q", out)
|
||||
}
|
||||
|
||||
got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{})
|
||||
if err != nil {
|
||||
@@ -37,4 +59,7 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing
|
||||
if len(got.Modules) != 0 || len(got.Unhostable) != 1 {
|
||||
t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable)
|
||||
}
|
||||
if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") {
|
||||
t.Errorf("the hold does not name the module and the capability: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,7 +246,7 @@ const kindVaultSilent = "vault-silent"
|
||||
// vaultSilentObservation is that, as a condition: `machine.<machine>.vault-silent`.
|
||||
func vaultSilentObservation(node string, b coreBoxes) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent,
|
||||
Machine: node, Severity: conditions.Warning, Source: "sends"}
|
||||
Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource}
|
||||
var said []string
|
||||
if len(b.notPlaced) > 0 {
|
||||
o.Severity = conditions.Urgent
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
)
|
||||
|
||||
// **No send but a person's `push <node>` and a gate's own put-back moves the mesh's own core to a build older
|
||||
// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a
|
||||
// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under
|
||||
// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans;
|
||||
// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then
|
||||
// sent the control node the controller build before it. The older binary read the row it could not serve and the
|
||||
// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR
|
||||
// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here,
|
||||
// without needing its trigger: whatever made the mesh's record older, the send keeps the build.
|
||||
//
|
||||
// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last
|
||||
// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or
|
||||
// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend
|
||||
// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by
|
||||
// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until
|
||||
// a send carries the record — a person's `push <node>` accepts the older build — or the record moves forward: a
|
||||
// newer merge, or `rebuild mesh-controller`.
|
||||
//
|
||||
// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits'
|
||||
// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's
|
||||
// current build by the same order, and the guard does not stop that move. A build that failed its gate is never
|
||||
// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own
|
||||
// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back
|
||||
// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's
|
||||
// schema already (issue 352).
|
||||
|
||||
// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record.
|
||||
const kindCoreDowngrade = "core-downgrade-kept"
|
||||
|
||||
// coreDowngradeSource is the condition's source: the sends, which raise and clear it.
|
||||
const coreDowngradeSource = "sends"
|
||||
|
||||
// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the
|
||||
// store and its seat's row, and the node-engine, which reads what the controller declares.
|
||||
var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine}
|
||||
|
||||
// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate.
|
||||
type askedBuild struct {
|
||||
asked time.Time
|
||||
failedGate bool
|
||||
}
|
||||
|
||||
// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when
|
||||
// the request time is not known), and whether it failed its gate — and false when the records hold none. A
|
||||
// variable, so a test reads its own order.
|
||||
var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) {
|
||||
builds, err := inv.Builds(ctx, module, askedLookBack)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Worked() && sameCommit(b.Commit, commit) {
|
||||
failed, err := inv.GateFailed(ctx, b.ID)
|
||||
if err != nil {
|
||||
return askedBuild{}, false, err
|
||||
}
|
||||
return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil
|
||||
}
|
||||
}
|
||||
return askedBuild{}, false, nil
|
||||
}
|
||||
|
||||
// askedLookBack is how many of a module's newest builds are read for when a build was asked.
|
||||
const askedLookBack = 50
|
||||
|
||||
// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record.
|
||||
type coreDowngrade struct {
|
||||
module, node, component string
|
||||
// sent is the build the machine was last sent, record the one the mesh holds.
|
||||
sent, record string
|
||||
sentAsked, recordAsked time.Time
|
||||
}
|
||||
|
||||
func (d coreDowngrade) String() string {
|
||||
return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+
|
||||
"keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+
|
||||
"(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record),
|
||||
d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node,
|
||||
d.module)
|
||||
}
|
||||
|
||||
// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine
|
||||
// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move
|
||||
// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure
|
||||
// but for asked.
|
||||
func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope,
|
||||
asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) {
|
||||
var out []coreDowngrade
|
||||
for module, was := range sent {
|
||||
component := coreComponent(module)
|
||||
now := record[module]
|
||||
if !guarded(component) || was == "" || now == "" || sameCommit(was, now) {
|
||||
continue
|
||||
}
|
||||
if scope.puttingBack && scope.modules[module] {
|
||||
continue
|
||||
}
|
||||
from, known, err := asked(module, was)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err)
|
||||
}
|
||||
if !known || from.failedGate {
|
||||
continue
|
||||
}
|
||||
to, known, err := asked(module, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err)
|
||||
}
|
||||
if known && to.asked.Before(from.asked) {
|
||||
out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now,
|
||||
sentAsked: from.asked, recordAsked: to.asked})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func guarded(component string) bool {
|
||||
for _, c := range downgradeGuarded {
|
||||
if c == component {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it
|
||||
// was last sent, module → commit, each said and raised as a condition.
|
||||
func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string,
|
||||
current map[string]inventory.CurrentBuild) (map[string]string, error) {
|
||||
record := map[string]string{}
|
||||
for m, c := range current {
|
||||
record[m] = c.Commit
|
||||
}
|
||||
found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) {
|
||||
return buildAsked(ctx, inv, module, commit)
|
||||
})
|
||||
if err != nil || len(found) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, d := range found {
|
||||
out[d.module] = d.sent
|
||||
sayCoreDowngrade(ctx, d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// coreDowngradeObservation is a kept core build as a condition: `core.<component>.<machine>.<kind>`.
|
||||
func coreDowngradeObservation(component, node, summary string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade,
|
||||
Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource,
|
||||
Summary: summary}
|
||||
}
|
||||
|
||||
func sayCoreDowngrade(ctx context.Context, d coreDowngrade) {
|
||||
fmt.Printf("KEPT: %s\n", d)
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return
|
||||
}
|
||||
if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil {
|
||||
fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err)
|
||||
}
|
||||
}
|
||||
|
||||
// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the
|
||||
// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record
|
||||
// moved forward — or did not carry at all. A send that kept the newer build leaves it open.
|
||||
func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) {
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil || carried == nil {
|
||||
return
|
||||
}
|
||||
var open []string
|
||||
for _, c := range downgradeGuarded {
|
||||
if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found {
|
||||
open = append(open, c)
|
||||
}
|
||||
}
|
||||
if len(open) == 0 {
|
||||
return
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err)
|
||||
return
|
||||
}
|
||||
for module, now := range current {
|
||||
component := coreComponent(module)
|
||||
if !guarded(component) {
|
||||
continue
|
||||
}
|
||||
if was, has := carried[module]; has && !sameCommit(was, now.Commit) {
|
||||
continue
|
||||
}
|
||||
key := coreDowngradeObservation(component, node, "").Key()
|
||||
if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -80,18 +80,17 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent, and applied: a holder is asked only once it has been (novox/hq issue 275).
|
||||
for _, node := range []string{"laptop", "anchor"} {
|
||||
pushedAndApplied(t, open, node)
|
||||
}
|
||||
|
||||
conn := onATestBus(t)
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
// Applied, and running: a holder is asked only once its module's have says it runs (novox/hq issue 275, ADR 0539 §9).
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
for _, run := range [][2]string{{"laptop", "keeper"}, {"laptop", "house"}, {"anchor", "keeper"}} {
|
||||
putRunningHave(t, js, run[0], run[1])
|
||||
}
|
||||
laptop, anchor := &fakeHolder{}, &fakeHolder{}
|
||||
laptop.serve(t, conn, "laptop")
|
||||
anchor.serve(t, conn, "anchor")
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
@@ -33,6 +34,14 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
nc, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
|
||||
MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(conn)
|
||||
@@ -52,8 +61,19 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
|
||||
resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
|
||||
}
|
||||
resources = append(resources, aFileResource("small", "tiny\n"))
|
||||
inline := sendable{Resources: resources}
|
||||
byDigest := sendable{fileReferences: true, Resources: resources}
|
||||
|
||||
bus := link.OverNATS{Conn: conn, JS: nc}
|
||||
body, err := inline.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
|
||||
t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
|
||||
}
|
||||
|
||||
bucket, err := openBucket(ctx, conn, "novox")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -72,14 +92,18 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// What the body names is exactly the three objects.
|
||||
namedSizes, err := namedObjects(refBody)
|
||||
if err != nil || len(namedSizes) != 3 {
|
||||
t.Fatalf("what the body names: %v %v", namedSizes, err)
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
|
||||
t.Fatalf("the declaration by reference: %v", err)
|
||||
}
|
||||
named := map[string]bool{}
|
||||
for name := range namedSizes {
|
||||
named[name] = true
|
||||
|
||||
// What the stream holds for the node is the reference form, and it names exactly the three objects.
|
||||
latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
|
||||
if err != nil || len(latest) != 1 {
|
||||
t.Fatalf("the latest declarations: %v %v", len(latest), err)
|
||||
}
|
||||
named, err := namedByLatest("novox", latest["novox"])
|
||||
if err != nil || len(named) != 3 {
|
||||
t.Fatalf("what the latest declaration names: %v %v", named, err)
|
||||
}
|
||||
novoxBucket, err := broker.DeclarationFilesBucketOf("novox")
|
||||
if err != nil {
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"time"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -56,7 +55,7 @@ func wireSizeOf(t *testing.T, s sendable) int {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The signature the controller's key makes is 64 bytes.
|
||||
msg, err := json.Marshal(link.Signed{Declaration: body, Signature: make([]byte, 64)})
|
||||
msg, err := link.Envelope(body, make([]byte, 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -244,6 +243,54 @@ func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
|
||||
// controller's grant does, and takes the second.
|
||||
type fallbackDelivery struct {
|
||||
bodies [][]byte
|
||||
first error
|
||||
}
|
||||
|
||||
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
|
||||
f.bodies = append(f.bodies, body)
|
||||
if len(f.bodies) == 1 && f.first != nil {
|
||||
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
|
||||
}
|
||||
return "digest", nil
|
||||
}
|
||||
|
||||
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
|
||||
// with every content inline and is made; a send that fails for another reason is not retried inline.
|
||||
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
|
||||
users := contentOf("users", 100<<10)
|
||||
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
|
||||
f := &fallbackDelivery{first: errors.New("timeout")}
|
||||
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
|
||||
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
|
||||
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
|
||||
}
|
||||
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
|
||||
t.Error("the first body should name the file by digest and the second carry it inline")
|
||||
}
|
||||
other := &fallbackDelivery{}
|
||||
other.first = nil
|
||||
failing := &failingDelivery{err: errors.New("the bus is down")}
|
||||
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
|
||||
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type failingDelivery struct {
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
|
||||
f.calls++
|
||||
return "", f.err
|
||||
}
|
||||
|
||||
// A bucket for the sweep.
|
||||
type sweepBucketFake struct {
|
||||
objects map[string]time.Time // zero: named at the last look
|
||||
@@ -307,43 +354,22 @@ func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { //
|
||||
}
|
||||
}
|
||||
|
||||
// What a node's compositions name is read from the values of its bucket `composed-<node>`: the files each carries by digest
|
||||
// (ADR 0539 §4). An unreadable one is an error and never "names none", which would mark an object a node needs.
|
||||
func TestTheObjectsACompositionNamesAreReadFromItsValue(t *testing.T) {
|
||||
content := contentOf("a", 70<<10)
|
||||
digest := sdkdecl.DigestOf([]byte(content))
|
||||
ref, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": digest,
|
||||
"size": len(content)})
|
||||
small := json.RawMessage(`{"id":"m.g","type":"file","module":"m","path":"/etc/g","content":"x"}`)
|
||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
value, err := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: fp,
|
||||
Resources: []json.RawMessage{ref, small}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
named, err := namedByCompositions("novox", [][]byte{value})
|
||||
name, _ := sdkdecl.ObjectName(digest)
|
||||
// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
|
||||
// never "names none", which would mark an object a node needs.
|
||||
func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
|
||||
body, _ := s.Body()
|
||||
msg, _ := link.Envelope(body, []byte("sig"))
|
||||
named, err := namedByLatest("novox", msg)
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
|
||||
if err != nil || len(named) != 1 || !named[name] {
|
||||
t.Fatalf("%v %v", named, err)
|
||||
}
|
||||
none, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "ace", Module: "m", FP: fp, Resources: []json.RawMessage{small}})
|
||||
if got, err := namedByCompositions("ace", [][]byte{none}); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a composition naming none: %v %v", got, err)
|
||||
if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 {
|
||||
t.Fatalf("a declaration naming none: %v %v", none, err)
|
||||
}
|
||||
if got, err := namedByCompositions("ace", nil); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a node with no composition: %v %v", got, err)
|
||||
}
|
||||
if _, err := namedByCompositions("novox", [][]byte{[]byte("not json")}); err == nil {
|
||||
t.Fatal("an unreadable composition was read as naming none")
|
||||
}
|
||||
// A node's objects are those of every composition it keeps: the five newest fingerprints of a module name their own.
|
||||
other := sdkdecl.DigestOf([]byte(contentOf("b", 80<<10)))
|
||||
ref2, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": other, "size": 80 << 10})
|
||||
older, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64),
|
||||
Resources: []json.RawMessage{ref2}})
|
||||
both, err := namedByCompositions("novox", [][]byte{value, older})
|
||||
if err != nil || len(both) != 2 {
|
||||
t.Fatalf("two fingerprints of one module name %v (%v), want both objects", both, err)
|
||||
if _, err := namedByLatest("novox", []byte("not json")); err == nil {
|
||||
t.Fatal("an unreadable latest declaration was read as naming none")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -476,3 +502,33 @@ func whileHeld(node string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The
|
||||
// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the
|
||||
// lock is held.
|
||||
func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9
|
||||
const node = "lock-send-node"
|
||||
bus := &aBusThatSays{published: make(chan string, 1)}
|
||||
bus.heldAtPublish = func() bool { return whileHeld(node) }
|
||||
send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}}
|
||||
release := lockFilesOf(node) // a sweep is judging this node's bucket
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case n := <-bus.published:
|
||||
t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-bus.published:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the send did not go on once the sweep let go")
|
||||
}
|
||||
if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") {
|
||||
t.Fatalf("the send did not reach the publication (a lock held at it): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,10 +2,65 @@ package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
|
||||
// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
|
||||
// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
|
||||
// the declaration is back under 75%. The limit is the connection's own, not a constant.
|
||||
|
||||
// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
|
||||
const kindDeclarationNearLimit = "declaration-near-limit"
|
||||
|
||||
// declarationRoomSource is the condition's source: the sends.
|
||||
const declarationRoomSource = "sends"
|
||||
|
||||
func declarationRoomObservation(node string, size, limit int64) conditions.Observation {
|
||||
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindDeclarationNearLimit,
|
||||
Token: kindDeclarationNearLimit, Machine: node, Severity: conditions.Warning, Source: declarationRoomSource}
|
||||
if size > limit {
|
||||
o.Severity = conditions.Urgent
|
||||
}
|
||||
o.Summary = fmt.Sprintf("%s's declaration is %d bytes on the wire, %d%% of the bus's largest message of %d bytes",
|
||||
node, size, size*100/limit, limit)
|
||||
if size > limit {
|
||||
o.Summary += ": over it, so it cannot be sent"
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func init() {
|
||||
link.DeclarationMeasured = func(ctx context.Context, node string, size, limit int64) {
|
||||
settleDeclarationRoom(ctx, conditionsFrom, node, size, limit)
|
||||
}
|
||||
}
|
||||
|
||||
// settleDeclarationRoom raises or clears the condition of a declaration's size. A nil keeper (a tool, not the
|
||||
// serving controller) says nothing.
|
||||
func settleDeclarationRoom(ctx context.Context, keeper *conditions.Keeper, node string, size, limit int64) {
|
||||
if keeper == nil || limit <= 0 {
|
||||
return
|
||||
}
|
||||
obs := declarationRoomObservation(node, size, limit)
|
||||
if float64(size) < link.NearTheLimit*float64(limit) {
|
||||
if _, err := keeper.Clear(ctx, obs.Key(), fmt.Sprintf("%s's declaration is %d bytes, under the limit's margin", node, size)); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", obs.Key(), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
fmt.Printf("NEAR THE LIMIT: %s\n", obs.Summary)
|
||||
if _, err := keeper.Observe(ctx, obs); err != nil {
|
||||
fmt.Printf("the condition %s could not be raised: %v\n", obs.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// compactKept writes the vault's export compact. The composition (the SDK's keptFile) writes it indented, and its 334
|
||||
// sealed secrets were 164,763 bytes of it against 138,474 compact: a fifth of the declaration's room, in whitespace,
|
||||
// base64-encoded again on the wire. Every reader parses JSON, so the spacing is nobody's. A resource that is not
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// The vault's export is sent compact (issue 1689): same document, no indentation, one rewrite.
|
||||
func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
|
||||
indented := "{\n \"export\": 1,\n \"kept\": [\n {\n \"name\": \"a\"\n }\n ]\n}\n"
|
||||
other := "{\n \"x\": 1\n}\n"
|
||||
in := []map[string]any{
|
||||
{"id": "mesh-vault.kept", "type": "file", "path": "/srv/vault/export.json", "mode": "0600", "content": indented},
|
||||
{"id": "mesh-vault.config", "type": "file", "path": "/srv/vault/config.json", "content": other},
|
||||
{"id": "other.kept", "type": "file", "path": "/srv/other/notes.json", "content": other},
|
||||
}
|
||||
out := compactKept(in)
|
||||
want := "{\"export\":1,\"kept\":[{\"name\":\"a\"}]}\n"
|
||||
if out[0]["content"] != want || out[0]["mode"] != "0600" || out[0]["path"] != "/srv/vault/export.json" {
|
||||
t.Fatalf("the export is %q (%v), expected %q with its mode and path kept", out[0]["content"], out[0], want)
|
||||
}
|
||||
if out[1]["content"] != other || out[2]["content"] != other {
|
||||
t.Fatalf("a file that is not the export was rewritten: %v %v", out[1], out[2])
|
||||
}
|
||||
if in[0]["content"] != indented {
|
||||
t.Fatalf("the composed resource was changed in place")
|
||||
}
|
||||
if again := compactKept(out); again[0]["content"] != want {
|
||||
t.Fatalf("compacting twice changed it: %q", again[0]["content"])
|
||||
}
|
||||
}
|
||||
|
||||
// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
|
||||
func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
const limit = 1000
|
||||
key := declarationRoomObservation("novox", 0, limit).Key()
|
||||
read := func() (conditions.Condition, bool) {
|
||||
t.Helper()
|
||||
c, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, found
|
||||
}
|
||||
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("a declaration at 74.9% raised a condition")
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
|
||||
c, found := read()
|
||||
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
|
||||
!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
|
||||
t.Fatalf("at 75%% the condition is %v %+v", found, c)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
|
||||
if c, _ := read(); c.Severity != conditions.Urgent {
|
||||
t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("the condition stayed after the declaration fell under 75%")
|
||||
}
|
||||
}
|
||||
@@ -9,22 +9,23 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **An object is removed a day after no composition of the node names it** (novox/hq ADR 0486 §3, ADR 0539 §4, issue 1696), so a
|
||||
// node that is asleep and applies its wants later still finds every object they name. What a node's compositions name is read
|
||||
// from its bucket `composed-<node>`, where the controller keeps each module's five newest fingerprints and the node's own: nothing
|
||||
// else is kept of them. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
|
||||
// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
|
||||
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
|
||||
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
|
||||
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
|
||||
// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
|
||||
// node's compositions alone. Objects are immutable and regenerable from the controller's state, so a removal loses nothing.
|
||||
// node's latest declaration alone. Objects are immutable and regenerable from the
|
||||
// controller's state, so a removal loses nothing.
|
||||
|
||||
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no composition of the node.
|
||||
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
|
||||
const declarationUnnamed = "unnamed-since"
|
||||
|
||||
// sweepEvery is how often the bucket is judged: the removal is exact to within this.
|
||||
@@ -91,124 +92,41 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// namedByCompositions is the name of every object the compositions of a node name: the files over 64 KiB each carries by
|
||||
// digest (ADR 0539 §4). A composition that cannot be read is an error, never "names none": an object the node still needs
|
||||
// would be marked for removal.
|
||||
func namedByCompositions(node string, values [][]byte) (map[string]bool, error) {
|
||||
named := map[string]bool{}
|
||||
for _, value := range values {
|
||||
c, err := sdkdelivery.DecodeComposed(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err)
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{"resources": c.Resources})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names, err := namedObjects(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err)
|
||||
}
|
||||
for name := range names {
|
||||
named[name] = true
|
||||
}
|
||||
// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
|
||||
// "names none": an object the node still needs would be marked for removal.
|
||||
func namedByLatest(node string, signed []byte) (map[string]bool, error) {
|
||||
var s link.Signed
|
||||
if err := json.Unmarshal(signed, &s); err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
names, err := namedObjects(s.Declaration)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
named := make(map[string]bool, len(names))
|
||||
for name := range names {
|
||||
named[name] = true
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// keptCompositionsOf are the values of every composition in a node's bucket `composed-<node>` a node may still apply:
|
||||
// each key's newest revision (each module's five newest fingerprints and the node's own, ADR 0539 §4), and any older
|
||||
// revision of it whose digest a want of the node names, since the node applies exactly the composition its want names
|
||||
// (clause 1) and a want may still name a revision a newer one replaced (the review's C1). A node with no bucket has none.
|
||||
func keptCompositionsOf(ctx context.Context, js jetstream.JetStream, node string) ([][]byte, error) {
|
||||
bucket, err := sdkdelivery.ComposedBucketFor(node)
|
||||
// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
|
||||
// no entry.
|
||||
func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the compositions of %s: %w", node, err)
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the compositions of %s: %w", node, err)
|
||||
}
|
||||
wanted, err := digestsWantedOn(ctx, js, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out [][]byte
|
||||
for _, key := range keys {
|
||||
history, err := kv.History(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
out := map[string][]byte{}
|
||||
for _, node := range nodes {
|
||||
msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the composition %s: %w", key, err)
|
||||
}
|
||||
out = append(out, keptRevisions(history, wanted)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keptRevisions are, of one key's history oldest first, the newest value when the key is not deleted, and every older
|
||||
// value whose composition's digest is wanted.
|
||||
func keptRevisions(history []jetstream.KeyValueEntry, wanted map[string]bool) [][]byte {
|
||||
var out [][]byte
|
||||
for i, e := range history {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
if i == len(history)-1 {
|
||||
out = append(out, e.Value())
|
||||
continue
|
||||
}
|
||||
c, err := sdkdelivery.DecodeComposed(e.Value())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if d, err := sdkdelivery.ComposedDigestOf(c); err == nil && wanted[d] {
|
||||
out = append(out, e.Value())
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// digestsWantedOn is the digest of every composition a want of the node names, its own want's among them; none when
|
||||
// the assigner's bucket is not there yet.
|
||||
func digestsWantedOn(ctx context.Context, js jetstream.JetStream, node string) (map[string]bool, error) {
|
||||
kv, err := js.KeyValue(ctx, sdkdelivery.AssignerWantsBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return map[string]bool{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the wants of %s: %w", node, err)
|
||||
}
|
||||
lister, err := kv.ListKeys(ctx, jetstream.MetaOnly())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the wants of %s: %w", node, err)
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for key := range lister.Keys() {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if n, _, err := sdkdelivery.ParseWantKey(key); err == nil && n == node {
|
||||
if w, err := sdkdelivery.DecodeWant(e.Value()); err == nil && w.Composed != "" {
|
||||
out[w.Composed] = true
|
||||
}
|
||||
} else if n, err := sdkdelivery.ParseWantNodeKey(key); err == nil && n == node {
|
||||
if w, err := sdkdelivery.DecodeNodeWant(e.Value()); err == nil {
|
||||
out[w.Composed] = true
|
||||
}
|
||||
return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
|
||||
}
|
||||
out[node] = msg.Data
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -341,13 +259,15 @@ func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, e
|
||||
return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
|
||||
}
|
||||
defer lockFilesOf(node)()
|
||||
values, err := keptCompositionsOf(working, js, node)
|
||||
latest, err := latestDeclarations(working, conn, []string{node})
|
||||
if err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
named, err := namedByCompositions(node, values)
|
||||
if err != nil {
|
||||
return filesSweep{}, err
|
||||
named := map[string]bool{}
|
||||
if body, ok := latest[node]; ok {
|
||||
if named, err = namedByLatest(node, body); err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
}
|
||||
return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
|
||||
}
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// 1945 (the review's C1): the sweep keeps the files of a key's newest composition and of every older revision a want
|
||||
// of the node still names, so a node told to apply a replaced revision finds its files; an older revision no want names,
|
||||
// and a deleted key, keep nothing.
|
||||
func TestTheSweepKeepsWhatAWantStillNames(t *testing.T) {
|
||||
revision := func(content string, rev uint64, op jetstream.KeyValueOp) fakeComposedEntry {
|
||||
c := sdkdelivery.Composed{Node: "ace", Module: "m", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64),
|
||||
Resources: []json.RawMessage{json.RawMessage(`{"id":"m.f","type":"file","module":"m","path":"/etc/m","content":"` + content + `"}`)}}
|
||||
raw, err := sdkdelivery.EncodeComposed(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fakeComposedEntry{key: "k", value: raw, rev: rev, op: op}
|
||||
}
|
||||
digestOfEntry := func(e fakeComposedEntry) string {
|
||||
c, _ := sdkdelivery.DecodeComposed(e.value)
|
||||
d, _ := sdkdelivery.ComposedDigestOf(c)
|
||||
return d
|
||||
}
|
||||
one, two, three := revision("one", 1, jetstream.KeyValuePut), revision("two", 2, jetstream.KeyValuePut), revision("three", 3, jetstream.KeyValuePut)
|
||||
history := []jetstream.KeyValueEntry{one, two, three}
|
||||
kept := keptRevisions(history, map[string]bool{digestOfEntry(one): true})
|
||||
if len(kept) != 2 || string(kept[0]) != string(one.value) || string(kept[1]) != string(three.value) {
|
||||
t.Errorf("kept %d revisions, want the wanted first and the newest", len(kept))
|
||||
}
|
||||
if kept := keptRevisions(history, map[string]bool{}); len(kept) != 1 {
|
||||
t.Errorf("with no want, kept %d revisions, want the newest alone", len(kept))
|
||||
}
|
||||
deleted := []jetstream.KeyValueEntry{one, fakeComposedEntry{key: "k", rev: 2, op: jetstream.KeyValueDelete}}
|
||||
if kept := keptRevisions(deleted, map[string]bool{}); len(kept) != 0 {
|
||||
t.Errorf("a deleted key kept %d revisions", len(kept))
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,8 +17,9 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the forge says of a change's manifests (novox/hq ADR 0502): read through the forge's module on the bus, once per
|
||||
// commit. Kept by release 6 for the pull request's declared config; the delivery plan's reads of it are gone.
|
||||
// What a change says of the manifests it touches, read from the forge (novox/hq ADR 0419 clause 3, the lead's settlement
|
||||
// of issue 977 question 1): the controller asks the forge for each changed module.json at the commit and at what it
|
||||
// replaced, once per commit. The composed check's result is not used: the plan is cut after the merge.
|
||||
|
||||
// changeSource is one change to a repository whose manifests are read: the commit after it and, when known, what it
|
||||
// replaced (a pull request's base commit or branch). An empty Old is the first parent of New: the merge's base.
|
||||
@@ -62,6 +63,24 @@ var manifestAt = func(ctx context.Context, owner, repo, file, ref string) (catal
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
// parentOf is the first parent of a commit, from the forge. A variable so a test needs no forge.
|
||||
var parentOf = func(ctx context.Context, owner, repo, commit string) (string, error) {
|
||||
var found struct {
|
||||
Result struct {
|
||||
Parents []struct {
|
||||
SHA string `json:"sha"`
|
||||
} `json:"parents"`
|
||||
} `json:"result"`
|
||||
}
|
||||
if _, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(commit), &found); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(found.Result.Parents) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return found.Result.Parents[0].SHA, nil
|
||||
}
|
||||
|
||||
func escapedPath(p string) string {
|
||||
parts := strings.Split(p, "/")
|
||||
for i := range parts {
|
||||
@@ -95,6 +114,12 @@ func askForgeApi(ctx context.Context, apiPath string, into any) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit.
|
||||
var (
|
||||
manifestsRead = map[string]manifestRead{}
|
||||
manifestsReadMu sync.Mutex
|
||||
)
|
||||
|
||||
type manifestRead struct {
|
||||
manifest catalogue.Manifest
|
||||
exists bool
|
||||
@@ -122,11 +147,96 @@ func manifestCached(ctx context.Context, owner, repo, file, ref string) (catalog
|
||||
return m, exists, err
|
||||
}
|
||||
|
||||
// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit.
|
||||
var (
|
||||
manifestsRead = map[string]manifestRead{}
|
||||
manifestsReadMu sync.Mutex
|
||||
)
|
||||
// verbsServed are the seat verbs a manifest serves, as "<seat>.<verb>": those of the seats it defines, those of the
|
||||
// seats it claims, and its own tools as "<module>.<tool>".
|
||||
func verbsServed(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, s := range m.DefinesSeats {
|
||||
for _, v := range s.Serves {
|
||||
out = appendNew(out, s.Name+"."+v.Name)
|
||||
}
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
for _, v := range c.ServesFor(m) {
|
||||
out = appendNew(out, c.Name+"."+v)
|
||||
}
|
||||
}
|
||||
for _, t := range m.Tools {
|
||||
out = appendNew(out, m.Module+"."+t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// verbsInvoked are the verbs a manifest calls, as "<seat>.<verb>" or "<module>.<tool>".
|
||||
func verbsInvoked(m catalogue.Manifest) []string {
|
||||
var out []string
|
||||
for _, i := range m.Invokes {
|
||||
if i = strings.TrimPrefix(i, "seat:"); i != "*" && i != "" {
|
||||
out = appendNew(out, i)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func added(now, was []string) []string {
|
||||
var out []string
|
||||
for _, v := range now {
|
||||
if !slices.Contains(was, v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// changesOf reads, for each manifest the changes touch, what they add to the verbs its module serves and invokes, by
|
||||
// module. A change whose paths are unsaid or truncated names no manifest: nothing is read for it. An error is the
|
||||
// forge's, never read as "no change": the plan is not cut on a guess.
|
||||
func changesOf(ctx context.Context, sources []changeSource) (map[string]moduleChange, error) {
|
||||
out := map[string]moduleChange{}
|
||||
for _, s := range sources {
|
||||
if s.Truncated {
|
||||
continue
|
||||
}
|
||||
for _, p := range s.Paths {
|
||||
if path.Base(p) != "module.json" || slices.Contains(s.Removed, p) {
|
||||
continue
|
||||
}
|
||||
now, exists, err := manifestCached(ctx, s.Owner, s.Repo, p, s.New)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !exists || now.Module == "" {
|
||||
continue
|
||||
}
|
||||
old := s.Old
|
||||
if old == "" {
|
||||
first := s.Since
|
||||
if first == "" {
|
||||
first = s.New
|
||||
}
|
||||
if old, err = parentOf(ctx, s.Owner, s.Repo, first); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var was catalogue.Manifest
|
||||
if old != "" {
|
||||
if was, _, err = manifestCached(ctx, s.Owner, s.Repo, p, old); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
c := out[now.Module]
|
||||
c.Manifest = true
|
||||
for _, v := range added(verbsServed(now), verbsServed(was)) {
|
||||
c.Serves = appendNew(c.Serves, v)
|
||||
}
|
||||
for _, v := range added(verbsInvoked(now), verbsInvoked(was)) {
|
||||
c.Invokes = appendNew(c.Invokes, v)
|
||||
}
|
||||
out[now.Module] = c
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// truncatedPaths are the manifests a change that names only a beginning of its files is read as changing: those of the
|
||||
// modules it moves that are built from its repository. Said, never skipped: no edge may be lost to a long list.
|
||||
@@ -141,3 +251,21 @@ func truncatedPaths(entries []inventory.Entry, owner, repo string, moved []strin
|
||||
owner, repo, strings.Join(out, ", "))
|
||||
return out
|
||||
}
|
||||
|
||||
// changeSourcesOf are a group's members as changes to read: each head over what its pull request merges into.
|
||||
func changeSourcesOf(members []orderMember, entries []inventory.Entry, reach map[string]orderReach) []changeSource {
|
||||
var out []changeSource
|
||||
for _, m := range members {
|
||||
p := m.pull()
|
||||
if p.PathsTruncated {
|
||||
p.Paths, p.PathsTruncated = truncatedPaths(entries, p.Owner, p.Repo, reach[m.ID].Moved), false
|
||||
}
|
||||
old := p.BaseSHA
|
||||
if old == "" {
|
||||
old = p.Base
|
||||
}
|
||||
out = append(out, changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old, Paths: p.Paths,
|
||||
Truncated: p.PathsTruncated, Removed: p.Removed})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table
|
||||
// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the
|
||||
// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery.<id>.stalled`; healer
|
||||
// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only
|
||||
// the next observation says whether it worked (ADR 0231).
|
||||
|
||||
// probeDeliveriesID is the probe that reads the delivery's owner.
|
||||
const probeDeliveriesID = "D14"
|
||||
|
||||
// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope.
|
||||
const kindDeliveryStalled = "stalled"
|
||||
|
||||
// deliveryOwnerWithin is how long the owner is given to answer.
|
||||
var deliveryOwnerWithin = 10 * time.Second
|
||||
|
||||
// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it.
|
||||
type stalledLine struct {
|
||||
ID string `json:"id"`
|
||||
// Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347).
|
||||
Number int `json:"number,omitempty"`
|
||||
State string `json:"state"`
|
||||
// Walk is the walk a line is about: a waiting walk no live delivery holds, or, asked with `walks`, each walk
|
||||
// waiting for the owner's word and what holds it (novox/hq issues 362, 529).
|
||||
Walk string `json:"walk,omitempty"`
|
||||
// Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each:
|
||||
// "mesh/check pending since <UTC time>", "mesh/check never set" (novox/hq issue 438).
|
||||
Waiting []string `json:"waiting,omitempty"`
|
||||
// Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq
|
||||
// issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before
|
||||
// says none, and Waiting is said as it reads.
|
||||
Checks []waitingCheck `json:"checks,omitempty"`
|
||||
// Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held
|
||||
// (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no
|
||||
// delivery, and from a mesh-delivery that does not say it yet; the words are never read for it.
|
||||
Held string `json:"held,omitempty"`
|
||||
For string `json:"for"`
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
// Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty
|
||||
// or anything else is a warning, as before.
|
||||
Severity string `json:"severity,omitempty"`
|
||||
// Needs is "new-head" for a delivery whose cure is its author's new head (novox/hq ADR 0505 §5), Author that author
|
||||
// as the operator reads it ("the author of feat/x", with the session that pushed it when its pull request names one)
|
||||
// and Branch its branch. A mesh-delivery from before says none, and the words are as before.
|
||||
Needs string `json:"needs,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
}
|
||||
|
||||
// needsNewHead is what a stalled line needs when its cure is its author's new head (ADR 0505 §5).
|
||||
const needsNewHead = "new-head"
|
||||
|
||||
// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise.
|
||||
func (l stalledLine) severity() conditions.Severity {
|
||||
if l.Severity == string(conditions.Urgent) {
|
||||
return conditions.Urgent
|
||||
}
|
||||
return conditions.Warning
|
||||
}
|
||||
|
||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||
// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say.
|
||||
type waitingCheck struct {
|
||||
Context string `json:"context"`
|
||||
State string `json:"state"`
|
||||
Since string `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces.
|
||||
var wordsNow = time.Now
|
||||
|
||||
// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its
|
||||
// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads.
|
||||
var wordsZone = func() *time.Location { return time.Local }
|
||||
|
||||
// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's.
|
||||
func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") }
|
||||
|
||||
// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted;
|
||||
// a seam a test replaces. The holder's own refusal is an error naming it.
|
||||
var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
granted := false
|
||||
for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner {
|
||||
granted = granted || v.Verb == verb
|
||||
}
|
||||
if !granted {
|
||||
return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
if conn == nil {
|
||||
return nil, errors.New("this controller is not on the bus")
|
||||
}
|
||||
answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error)
|
||||
}
|
||||
return unwrapToolResult(answer.Result), nil
|
||||
}
|
||||
|
||||
// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the
|
||||
// protocol's content list holding it as text.
|
||||
func unwrapToolResult(raw json.RawMessage) json.RawMessage {
|
||||
var wrapped struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) {
|
||||
return json.RawMessage(wrapped.Content[0].Text)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record,
|
||||
// or none answers (D3 says that one).
|
||||
func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) {
|
||||
if !held {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{})
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return nil, nil // the holder not running is D3's holder-silent, said once there
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lines []stalledLine
|
||||
if err := json.Unmarshal(raw, &lines); err != nil {
|
||||
return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err)
|
||||
}
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// walkWaits are, by walk, the lines mesh-delivery says of a walk waiting for its word: what holds it.
|
||||
type walkWaits map[string][]stalledLine
|
||||
|
||||
// walksWaiting asks the delivery's owner what holds each walk waiting for its word (novox/hq issue 529): its
|
||||
// `stalled` asked with `walks`, so a line of `plans` names a held delivery, its reason and the verb that releases
|
||||
// it, instead of pointing at a hand start. Nil when no holder is on record or none answers: the line then reads as
|
||||
// before. An owner from before this answers its stalled lines, of which only those naming a walk are kept.
|
||||
func walksWaiting(ctx context.Context, conn *nats.Conn, held bool) walkWaits {
|
||||
if !held {
|
||||
return nil
|
||||
}
|
||||
raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{"walks": true})
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var lines []stalledLine
|
||||
if json.Unmarshal(raw, &lines) != nil {
|
||||
return nil
|
||||
}
|
||||
out := walkWaits{}
|
||||
for _, l := range lines {
|
||||
if l.Walk != "" {
|
||||
out[l.Walk] = append(out[l.Walk], l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stalledObservations are the conditions of what is stalled.
|
||||
func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
||||
Severity: l.severity(),
|
||||
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
||||
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
||||
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
||||
if l.operatorsOnly() {
|
||||
o.Resolver = conditions.ResolverOperator
|
||||
}
|
||||
o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o)
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeDeliveries is D14: every delivery held past its state's bound is said.
|
||||
func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
entries, err := d.open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var conn *nats.Conn
|
||||
if d.js != nil {
|
||||
conn = d.js.Conn()
|
||||
}
|
||||
lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stalledObservations(lines), nil
|
||||
}
|
||||
|
||||
// --- H2, for a delivery ---------------------------------------------------------------------------------
|
||||
|
||||
// connOf is the bus a healer asks over.
|
||||
func (h *healing) connOf() *nats.Conn {
|
||||
if h.js == nil {
|
||||
return nil
|
||||
}
|
||||
return h.js.Conn()
|
||||
}
|
||||
|
||||
// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table
|
||||
// lets H2 take; never one whose state is the operator's.
|
||||
func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) {
|
||||
lines, err := deliveriesStalled(ctx, h.connOf(), true)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
}
|
||||
for _, l := range lines {
|
||||
if l.ID != c.Subject.ID {
|
||||
continue
|
||||
}
|
||||
if l.operatorsOnly() {
|
||||
return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil
|
||||
}
|
||||
return c.Key, true, "", nil
|
||||
}
|
||||
return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil
|
||||
}
|
||||
|
||||
// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the
|
||||
// transition its table names. A refusal is no repair, said; the next observation says whether it worked.
|
||||
func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) {
|
||||
raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key})
|
||||
if err != nil {
|
||||
if errors.Is(err, link.ErrNothingServes) {
|
||||
return "", "", err
|
||||
}
|
||||
return "closed nothing", oneLine(err.Error()), nil
|
||||
}
|
||||
var said string
|
||||
if json.Unmarshal(raw, &said) != nil {
|
||||
said = string(raw)
|
||||
}
|
||||
return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's
|
||||
// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`.
|
||||
|
||||
// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records
|
||||
// what close was asked.
|
||||
func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string {
|
||||
t.Helper()
|
||||
var closed []string
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
switch verb {
|
||||
case "stalled":
|
||||
raw, _ := json.Marshal(lines)
|
||||
return raw, nil
|
||||
case "close":
|
||||
closed = append(closed, args["id"].(string))
|
||||
if closeErr != nil {
|
||||
return nil, closeErr
|
||||
}
|
||||
return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil
|
||||
}
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
return nil, nil
|
||||
}
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
return &closed
|
||||
}
|
||||
|
||||
func holdTheDeliverySeat(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "mesh-delivery", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var twoStalled = []stalledLine{
|
||||
{ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s",
|
||||
H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"},
|
||||
{ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s",
|
||||
H2: "none: the state is the operator's", Says: "it waits for the operator"},
|
||||
}
|
||||
|
||||
func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
ownerAnswers(t, twoStalled, nil)
|
||||
d := &doctor{open: open}
|
||||
got, err := probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("with no holder on record D14 said %+v %v", got, err)
|
||||
}
|
||||
holdTheDeliverySeat(t, open)
|
||||
got, err = probeDeliveries(ctx, d)
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("D14 said %+v %v", got, err)
|
||||
}
|
||||
if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning ||
|
||||
got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") {
|
||||
t.Fatalf("the first is %+v", got[0])
|
||||
}
|
||||
if got[1].Resolver != conditions.ResolverOperator {
|
||||
t.Fatalf("a held delivery is not the operator's: %+v", got[1])
|
||||
}
|
||||
// The holder not running is D3's to say, not D14's.
|
||||
was := askDeliveryOwner
|
||||
askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) {
|
||||
return nil, link.ErrNothingServes
|
||||
}
|
||||
defer func() { askDeliveryOwner = was }()
|
||||
if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 {
|
||||
t.Fatalf("an owner that is down was said by D14: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a
|
||||
// refusal is no repair.
|
||||
func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
h, told, _ := healingOn(t, open)
|
||||
closed := ownerAnswers(t, twoStalled, nil)
|
||||
for _, o := range stalledObservations(twoStalled) {
|
||||
o.Source = probeDeliveriesID
|
||||
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
h.tick(ctx)
|
||||
if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) {
|
||||
t.Fatalf("close was asked for %v", *closed)
|
||||
}
|
||||
acts := told.said()
|
||||
if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" {
|
||||
t.Fatalf("said %+v", acts)
|
||||
}
|
||||
// A refusal: closed nothing, said so.
|
||||
c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled")
|
||||
ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering"))
|
||||
act, said, err := repairDelivery(ctx, h, c)
|
||||
if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") {
|
||||
t.Fatalf("a refused close reads %q %q %v", act, said, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else
|
||||
// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read.
|
||||
func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) {
|
||||
granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And release and stop, which the operator's warrant chooses (novox/hq ADR 0259).
|
||||
for _, verb := range []string{"stalled", "close", "release", "stop"} {
|
||||
if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) {
|
||||
t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb)
|
||||
}
|
||||
}
|
||||
if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") {
|
||||
t.Fatalf("a verb the grant does not name was asked: %v", err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if _, err := deliveriesStalled(t.Context(), conn, true); err != nil {
|
||||
t.Fatalf("an owner not running is D3's, not an error: %v", err)
|
||||
}
|
||||
lines, _ := json.Marshal(twoStalled)
|
||||
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}})
|
||||
sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) {
|
||||
body, _ := json.Marshal(link.Answer{Result: wrapped})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
got, err := deliveriesStalled(ctx, conn, true)
|
||||
if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" {
|
||||
t.Fatalf("over the bus: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The order of a plan's modules is read from the module graph and from what the change says of the manifests
|
||||
// (novox/hq ADR 0419), per module and never per repository: no `after:` line is read, and no member of a group
|
||||
// is ordered against another. Pure, so a test needs no store.
|
||||
|
||||
// The rules that draw an edge of a plan's order, in the words the plan shows.
|
||||
const (
|
||||
ruleVerbBeforeCaller = "verb before caller"
|
||||
ruleVersionSkew = "version skew"
|
||||
ruleEngineFirst = "engine before controller"
|
||||
)
|
||||
|
||||
// The two modules the rollout rules name.
|
||||
const (
|
||||
theController = "mesh-controller"
|
||||
theEngine = "mesh-host"
|
||||
)
|
||||
|
||||
// moduleChange is what a change says of one module's manifest: the seat verbs ("<seat>.<verb>") it adds to what the
|
||||
// module serves, the seat verbs it adds to what the module invokes, and whether the manifest changes at all.
|
||||
type moduleChange struct {
|
||||
Serves []string
|
||||
Invokes []string
|
||||
Manifest bool
|
||||
}
|
||||
|
||||
// planOrder is the edges the rules draw among a plan's modules, and the cycle they make when they make one.
|
||||
type planOrder struct {
|
||||
Edges []inventory.Edge
|
||||
// Cycle is every edge of a cycle among the order's edges and the graph's, the module graph's included.
|
||||
Cycle []inventory.Edge
|
||||
}
|
||||
|
||||
// CycleSaid names every module and edge of the cycle, for a check to fail with (ADR 0419 clause 6).
|
||||
func (o planOrder) CycleSaid() string {
|
||||
if len(o.Cycle) == 0 {
|
||||
return ""
|
||||
}
|
||||
var parts []string
|
||||
for _, e := range o.Cycle {
|
||||
parts = append(parts, edgeSaid(e))
|
||||
}
|
||||
return "the modules of the plan are ordered in a cycle: " + strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// edgeSaid is an edge as a person reads it: the module after, the module before, and why.
|
||||
func edgeSaid(e inventory.Edge) string {
|
||||
why := e.Why
|
||||
if why == "" {
|
||||
why = e.Kind
|
||||
}
|
||||
return fmt.Sprintf("%s after %s (%s)", e.From, e.To, why)
|
||||
}
|
||||
|
||||
// orders is whether an edge of the module graph orders a plan's tiers (tiersOf).
|
||||
func orders(e inventory.Edge) bool {
|
||||
return e.Kind != inventory.EdgePackages && e.Kind != inventory.EdgeBuiltBy && e.From != e.To
|
||||
}
|
||||
|
||||
// orderEdges draws the edges of a plan's order among its moved modules (ADR 0419 clause 3, 4): a module whose change
|
||||
// adds a verb to a seat goes before one whose change adds a call of that verb; the controller, when the plan moves
|
||||
// it, goes before every module whose manifest the plan changes, the node-engine excepted; and the node-engine goes
|
||||
// before the controller by default, when the plan moves both and no edge orders them either way, transitively. An
|
||||
// edge is "From after To". graph is the module graph, whose edges order too and are never drawn again.
|
||||
func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge) planOrder {
|
||||
var out planOrder
|
||||
sorted := append([]string(nil), moved...)
|
||||
sort.Strings(sorted)
|
||||
for _, server := range sorted {
|
||||
for _, caller := range sorted {
|
||||
if server == caller {
|
||||
continue
|
||||
}
|
||||
for _, verb := range changes[server].Serves {
|
||||
if slices.Contains(changes[caller].Invokes, verb) {
|
||||
out.Edges = append(out.Edges, inventory.Edge{From: caller, To: server, Kind: edgeChangeOrder,
|
||||
Why: ruleVerbBeforeCaller + ": " + verb})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(sorted, theController) {
|
||||
for _, m := range sorted {
|
||||
if m != theController && m != theEngine && changes[m].Manifest {
|
||||
out.Edges = append(out.Edges, inventory.Edge{From: m, To: theController, Kind: edgeChangeOrder,
|
||||
Why: ruleVersionSkew})
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) {
|
||||
all := orderingEdges(graph, out.Edges)
|
||||
switch {
|
||||
case pathBetween(all, theController, theEngine) != nil:
|
||||
// An edge already puts the controller after the engine: the default says the same.
|
||||
case pathBetween(all, theEngine, theController) != nil:
|
||||
// An edge puts the engine after the controller: it overrules the default, and says so.
|
||||
path := pathBetween(all, theEngine, theController)
|
||||
for i := range out.Edges {
|
||||
if out.Edges[i] == path[0] {
|
||||
out.Edges[i].Why += " (overrules " + ruleEngineFirst + ")"
|
||||
break
|
||||
}
|
||||
}
|
||||
default:
|
||||
out.Edges = append(out.Edges, inventory.Edge{From: theController, To: theEngine, Kind: edgeChangeOrder,
|
||||
Why: ruleEngineFirst})
|
||||
}
|
||||
}
|
||||
all := orderingEdges(graph, out.Edges)
|
||||
for _, e := range out.Edges {
|
||||
if back := pathBetween(all, e.To, e.From); back != nil {
|
||||
out.Cycle = append(out.Cycle, e)
|
||||
out.Cycle = append(out.Cycle, back...)
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// orderingEdges is the graph's edges that order and the edges drawn, together.
|
||||
func orderingEdges(graph, drawn []inventory.Edge) []inventory.Edge {
|
||||
var out []inventory.Edge
|
||||
for _, e := range graph {
|
||||
if orders(e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return append(out, drawn...)
|
||||
}
|
||||
|
||||
// pathBetween is the edges that lead from a module to another along "depends on" (From to To), the shortest, or nil
|
||||
// when the first does not depend on the second.
|
||||
func pathBetween(edges []inventory.Edge, from, to string) []inventory.Edge {
|
||||
if from == to {
|
||||
return nil
|
||||
}
|
||||
via := map[string]inventory.Edge{}
|
||||
seen := map[string]bool{from: true}
|
||||
queue := []string{from}
|
||||
for len(queue) > 0 {
|
||||
at := queue[0]
|
||||
queue = queue[1:]
|
||||
for _, e := range edges {
|
||||
if e.From != at || seen[e.To] {
|
||||
continue
|
||||
}
|
||||
seen[e.To] = true
|
||||
via[e.To] = e
|
||||
if e.To == to {
|
||||
var path []inventory.Edge
|
||||
for m := to; m != from; m = via[m].From {
|
||||
path = append([]inventory.Edge{via[m]}, path...)
|
||||
}
|
||||
return path
|
||||
}
|
||||
queue = append(queue, e.To)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The order of a plan's modules is read from the module graph and the manifests' change alone (novox/hq ADR 0419):
|
||||
// no `after:` line, no member of a group. Pure, like orderOf was.
|
||||
|
||||
func hasEdge(edges []inventory.Edge, from, to string) (inventory.Edge, bool) {
|
||||
for _, e := range edges {
|
||||
if e.From == from && e.To == to {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Edge{}, false
|
||||
}
|
||||
|
||||
// A module whose change adds a verb to a seat goes before the module whose change adds a call of it: the caller
|
||||
// depends on the server, the plan puts them in two tiers, and the edge says why.
|
||||
func TestAVerbIsServedBeforeItsCallerIsSent(t *testing.T) {
|
||||
changes := map[string]moduleChange{
|
||||
"server": {Serves: []string{"mesh-delivery.groups"}, Manifest: true},
|
||||
"caller": {Invokes: []string{"mesh-delivery.groups"}, Manifest: true},
|
||||
}
|
||||
got := orderEdges([]string{"caller", "server"}, changes, nil)
|
||||
e, ok := hasEdge(got.Edges, "caller", "server")
|
||||
if !ok || !strings.Contains(e.Why, "verb before caller") || !strings.Contains(e.Why, "mesh-delivery.groups") {
|
||||
t.Fatalf("the caller does not follow the server by the verb: %+v", got.Edges)
|
||||
}
|
||||
if len(got.Cycle) != 0 {
|
||||
t.Fatalf("a cycle: %+v", got.Cycle)
|
||||
}
|
||||
if tiers := tiersOf([]string{"caller", "server"}, got.Edges); len(tiers) != 2 || tiers[0][0] != "server" {
|
||||
t.Fatalf("tiers %v, want the server first", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// Edges of one rule both ways are a cycle: every module and edge is named.
|
||||
func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) {
|
||||
changes := map[string]moduleChange{
|
||||
"x": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}},
|
||||
"y": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}},
|
||||
}
|
||||
got := orderEdges([]string{"x", "y"}, changes, nil)
|
||||
if len(got.Cycle) < 2 {
|
||||
t.Fatalf("no cycle found: %+v", got)
|
||||
}
|
||||
said := got.CycleSaid()
|
||||
for _, want := range []string{"x", "y", "s.a", "s.b"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("the cycle is said as %q, without %s", said, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine goes before the controller by default, and says so; an edge of the manifests the other way
|
||||
// overrules it, and the edge that did is said.
|
||||
func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T) {
|
||||
got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil)
|
||||
e, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host")
|
||||
if !ok || e.Why != "engine before controller" {
|
||||
t.Fatalf("the engine does not go first: %+v", got.Edges)
|
||||
}
|
||||
changes := map[string]moduleChange{
|
||||
"mesh-host": {Serves: []string{"s.v"}},
|
||||
"mesh-controller": {Invokes: []string{"s.v"}},
|
||||
}
|
||||
got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
|
||||
if _, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host"); !ok || len(got.Edges) != 1 ||
|
||||
!strings.Contains(got.Edges[0].Why, "verb before caller") {
|
||||
t.Fatalf("the verb edge does not place the controller after the engine: %+v", got.Edges)
|
||||
}
|
||||
// The other way: the engine calls what the controller brings.
|
||||
changes = map[string]moduleChange{
|
||||
"mesh-controller": {Serves: []string{"s.v"}},
|
||||
"mesh-host": {Invokes: []string{"s.v"}},
|
||||
}
|
||||
got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil)
|
||||
e, ok = hasEdge(got.Edges, "mesh-host", "mesh-controller")
|
||||
if !ok || len(got.Edges) != 1 || !strings.Contains(e.Why, "overrules engine before controller") {
|
||||
t.Fatalf("the verb edge does not say it overruled the default: %+v", got.Edges)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller goes before every module whose manifest the plan changes (version skew), the node-engine excepted.
|
||||
func TestTheControllerGoesBeforeEveryChangedManifest(t *testing.T) {
|
||||
changes := map[string]moduleChange{"app": {Manifest: true}, "other": {}}
|
||||
got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil)
|
||||
if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" {
|
||||
t.Fatalf("the changed manifest does not follow the controller: %+v", got.Edges)
|
||||
}
|
||||
if _, ok := hasEdge(got.Edges, "other", "mesh-controller"); ok {
|
||||
t.Fatalf("a module with an unchanged manifest was ordered: %+v", got.Edges)
|
||||
}
|
||||
if got := orderEdges([]string{"app", "other"}, changes, nil); len(got.Edges) != 0 {
|
||||
t.Fatalf("version skew ordered a plan that does not move the controller: %+v", got.Edges)
|
||||
}
|
||||
}
|
||||
|
||||
// The order is per module, not per repository: of two modules of one repository only the one with the edge moves.
|
||||
func TestTheOrderIsPerModuleNotPerRepository(t *testing.T) {
|
||||
changes := map[string]moduleChange{
|
||||
"a": {Serves: []string{"s.v"}},
|
||||
"c": {Invokes: []string{"s.v"}},
|
||||
}
|
||||
got := orderEdges([]string{"a", "b", "c"}, changes, nil)
|
||||
tiers := tiersOf([]string{"a", "b", "c"}, got.Edges)
|
||||
if len(tiers) != 2 || strings.Join(tiers[0], ",") != "a,b" || strings.Join(tiers[1], ",") != "c" {
|
||||
t.Fatalf("tiers %v, want [a b] then [c]", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// The edges a plan's order was drawn with are kept on the plan, and its gates read them: a later tier waits for the
|
||||
// earlier to be applied although the catalogue's graph holds no such edge (ADR 0419, the lead's settlement Q4).
|
||||
func TestAPlansOwnOrderGatesItsTiersWhateverTheStoreGraphHolds(t *testing.T) {
|
||||
p := inventory.Plan{Tiers: [][]string{{"server"}, {"caller"}}, Tier: 0,
|
||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "caller", To: "server", Kind: edgeChangeOrder,
|
||||
Why: ruleVerbBeforeCaller + ": s.v"}}}}
|
||||
if got := gates(p, nil, func(string) bool { return true }); len(got) != 1 || got[0] != "server" {
|
||||
t.Fatalf("the caller's tier waits for %v; want the server", got)
|
||||
}
|
||||
p.Delivery.Order = nil
|
||||
if got := gates(p, nil, func(string) bool { return true }); len(got) != 0 {
|
||||
t.Fatalf("a plan with no order waits for %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A verb an earlier merge of the same repository in the batch added is added by the batch: the change is compared with
|
||||
// what the repository held before the batch, not with the first parent of its latest merge.
|
||||
func TestAVerbAnEarlierMergeOfTheBatchAddedDrawsItsEdge(t *testing.T) {
|
||||
was, wasParent := manifestAt, parentOf
|
||||
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
const first, second, before = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "0000000000000000000000000000000000000000"
|
||||
parentOf = func(_ context.Context, _, _, commit string) (string, error) {
|
||||
if commit == first {
|
||||
return before, nil
|
||||
}
|
||||
return first, nil // the latest merge's parent holds the verb already
|
||||
}
|
||||
manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) {
|
||||
m := catalogue.Manifest{Module: "server"}
|
||||
if ref != before {
|
||||
m.DefinesSeats = []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "v"}}}}
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
src := changeSource{Owner: "novox", Repo: "r", New: second, Paths: []string{"module.json"}}
|
||||
if got, err := changesOf(context.Background(), []changeSource{src}); err != nil || len(got["server"].Serves) != 0 {
|
||||
t.Fatalf("against the first parent of the latest merge: %+v %v", got, err)
|
||||
}
|
||||
src.Since = first
|
||||
got, err := changesOf(context.Background(), []changeSource{src})
|
||||
if err != nil || len(got["server"].Serves) != 1 || got["server"].Serves[0] != "s.v" {
|
||||
t.Fatalf("against what the batch found: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A change that names only a beginning of its files is read as changing the manifests of the modules it moves.
|
||||
func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T) {
|
||||
entries := []inventory.Entry{
|
||||
{Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}},
|
||||
{Manifest: catalogue.Manifest{Module: "b"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/b"}},
|
||||
{Manifest: catalogue.Manifest{Module: "c"}, Source: inventory.Source{Repository: "novox/other", Path: "modules/c"}},
|
||||
}
|
||||
got := truncatedPaths(entries, "novox", "mesh-catalog", []string{"a", "c"})
|
||||
if len(got) != 1 || got[0] != "modules/a/module.json" {
|
||||
t.Fatalf("read as changed: %v", got)
|
||||
}
|
||||
members := []orderMember{{ID: "m", Repository: "novox/mesh-catalog", Head: "h", PathsTruncated: true, Paths: []string{"x"}}}
|
||||
src := changeSourcesOf(members, entries, map[string]orderReach{"m": {Moved: []string{"a"}}})
|
||||
if len(src) != 1 || src[0].Truncated || len(src[0].Paths) != 1 || src[0].Paths[0] != "modules/a/module.json" {
|
||||
t.Fatalf("the member's change is read as %+v", src)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksRecorded(t)
|
||||
withConditionsInMemory(t)
|
||||
for _, name := range []string{"app", "mesh-delivery"} {
|
||||
m := catalogue.Manifest{Module: name, Version: "1"}
|
||||
if name == "mesh-delivery" {
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
merge := func(commit string, paths ...string) inventory.Plan {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
|
||||
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 1)
|
||||
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
|
||||
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
|
||||
}
|
||||
return recent[0]
|
||||
}
|
||||
|
||||
// finish ends a walk as done, so the next merge has nothing of it to take over.
|
||||
finish := func(id string) {
|
||||
t.Helper()
|
||||
w, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.State = inventory.PlanDone
|
||||
if err := inv.SavePlan(ctx, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// heardOf has this mesh's own view of the plans take in the plan as the bucket holds it. A verb of the
|
||||
// controller (deliveryCommand, plansCommand) opens an inventory of its own and writes the plan there; this
|
||||
// mesh's inventory learns of the write from the bucket's watch, a moment later, and a loaded machine makes the
|
||||
// moment long enough for advanceHeld to read the plan as it was, still waiting for its word, and ask nothing
|
||||
// (issue 1399). Reading the plan from the bucket puts it into the view at once, with no wait to bound.
|
||||
heardOf := func(id string) {
|
||||
t.Helper()
|
||||
if _, err := inv.PlanByID(ctx, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No holder on record: the merge starts its walk, as before.
|
||||
// A cut takes its first tier from the builder's keys and a step of the walk records the take (the walk asks no
|
||||
// build), so each walk that starts is stepped once before its takes are counted.
|
||||
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
||||
advanceHeld(ctx, open)
|
||||
if p.Waiting() || len(*asked) != 1 {
|
||||
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
|
||||
// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
|
||||
finish(p.ID)
|
||||
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p = merge("c2bbbbbbbb", "modules/app/index.ts")
|
||||
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
|
||||
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
|
||||
}
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
|
||||
}
|
||||
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
|
||||
t.Errorf("a waiting walk reads %q", line)
|
||||
}
|
||||
|
||||
// The delivery's word starts it.
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
||||
t.Fatal("a walk was let go twice")
|
||||
}
|
||||
heardOf(p.ID)
|
||||
advanceHeld(ctx, open)
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the word did not start the walk: %v", *asked)
|
||||
}
|
||||
got, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
|
||||
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
||||
}
|
||||
|
||||
// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
|
||||
// ADR 0276): the merge's batch is cut once it ended.
|
||||
finish(p.ID)
|
||||
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
||||
advanceHeld(ctx, open)
|
||||
if p.Waiting() || len(*asked) != 3 {
|
||||
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
||||
}
|
||||
|
||||
// The owner down: a person starts a waiting walk, with why.
|
||||
finish(p.ID)
|
||||
p = merge("c4dddddddd", "modules/app/index.ts")
|
||||
if !p.Waiting() {
|
||||
t.Fatal("the walk did not wait")
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a walk was started by hand without why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heardOf(p.ID)
|
||||
advanceHeld(ctx, open)
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
|
||||
t.Fatalf("a person's word did not start the walk: %+v, asked %v, state %q, tier %d, note %q", got.Delivery, *asked, got.State, got.Tier, got.Note)
|
||||
}
|
||||
|
||||
// The owner's stop: failed, said as stopped by it.
|
||||
finish(p.ID)
|
||||
p = merge("c5eeeeeeee", "modules/app/index.ts")
|
||||
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
|
||||
"mesh-delivery for jochen"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ = inv.PlanByID(ctx, p.ID)
|
||||
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
|
||||
!strings.Contains(got.Note, "the operator stopped it") {
|
||||
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
|
||||
}
|
||||
|
||||
// Unassigned: the mesh is back on the controller's own path.
|
||||
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asks := len(*asked)
|
||||
p = merge("c6ffffffff", "modules/app/index.ts")
|
||||
advanceHeld(ctx, open)
|
||||
if p.Waiting() || len(*asked) != asks+1 {
|
||||
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 4)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
|
||||
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said inventory.Plan
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
|
||||
said.Delivery.Go == nil {
|
||||
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a walk let go by a verb was not said")
|
||||
}
|
||||
if checkEvents != nil || inventory.PlanSaved != nil {
|
||||
t.Fatal("the command's own bus was left in place")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
|
||||
type recordingDelivery struct {
|
||||
did []string
|
||||
grantErr error
|
||||
declareErr error
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
|
||||
for _, s := range sending {
|
||||
r.did = append(r.did, "grant "+s.node)
|
||||
}
|
||||
return r.grantErr
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
|
||||
if r.declareErr != nil {
|
||||
return "", r.declareErr
|
||||
}
|
||||
r.did = append(r.did, "declare "+s.node)
|
||||
return "digest-" + s.node, nil
|
||||
}
|
||||
|
||||
func ready(names ...string) []readyNode {
|
||||
var out []readyNode
|
||||
for _, n := range names {
|
||||
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
|
||||
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
|
||||
// carries the controller's own right to issue them, and goes first.
|
||||
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
|
||||
d := &recordingDelivery{}
|
||||
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
|
||||
if !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
|
||||
}
|
||||
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
|
||||
t.Fatalf("the digests sent were not answered: %v", digests)
|
||||
}
|
||||
|
||||
held := &recordingDelivery{}
|
||||
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
|
||||
if !reflect.DeepEqual(held.did, want) {
|
||||
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
|
||||
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
|
||||
// grant that would let the controller issue memberships is never held behind them.
|
||||
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
|
||||
// A failure naming no machine (the buckets): everything but the bus's machine.
|
||||
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
|
||||
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
|
||||
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
|
||||
!strings.Contains(err.Error(), "anchor, laptop not sent") {
|
||||
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
|
||||
}
|
||||
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
|
||||
}
|
||||
|
||||
// A membership that failed for one machine: that machine alone.
|
||||
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
|
||||
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
|
||||
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
|
||||
t.Fatalf("one machine's refused membership was not said: %v", err)
|
||||
}
|
||||
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
|
||||
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
|
||||
}
|
||||
|
||||
// The announced upgrade that hit it is asked again.
|
||||
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
|
||||
t.Fatal("an announcement whose send stopped at its grants is not asked again")
|
||||
}
|
||||
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
|
||||
t.Fatal("any failure is asked again, not only a grant's")
|
||||
}
|
||||
|
||||
// Nothing to send is nothing granted either.
|
||||
none := &recordingDelivery{grantErr: errors.New("never asked")}
|
||||
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
|
||||
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Whether the bus's machine goes first is read from the user list alone, by its digest.
|
||||
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
list := "users: [a, b]"
|
||||
if userListBehind(list, digestOf([]byte(list))) {
|
||||
t.Fatal("the list it was sent reads as behind")
|
||||
}
|
||||
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
|
||||
t.Fatal("a changed or never-sent list reads as current")
|
||||
}
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
// A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353).
|
||||
if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) {
|
||||
t.Fatal("a list nobody could compose reads as current")
|
||||
}
|
||||
if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") {
|
||||
t.Fatal("listBehind does not read the digest as userListBehind does")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
// checked against. Among the machines it is moved to the front; not among them it is added only
|
||||
// when it is behind.
|
||||
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
names []string
|
||||
holder string
|
||||
behind bool
|
||||
want []string
|
||||
}{
|
||||
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
|
||||
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
|
||||
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
|
||||
} {
|
||||
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
func severityRaisedFor(t *testing.T, line string) conditions.Severity {
|
||||
t.Helper()
|
||||
var l stalledLine
|
||||
if err := json.Unmarshal([]byte(line), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := stalledObservations([]stalledLine{l})
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%d conditions", len(got))
|
||||
}
|
||||
return got[0].Severity
|
||||
}
|
||||
|
||||
func TestAnUrgentStalledLineRaisesAnUrgentCondition(t *testing.T) { // 1278/AC1
|
||||
line := `{"id":"novox/mesh-delivery#5","state":"held","for":"3m0s","bound":"0s","h2":"close","says":"held","severity":"urgent"}`
|
||||
if got := severityRaisedFor(t, line); got != conditions.Urgent {
|
||||
t.Errorf("an urgent line raised %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStalledLineWithoutOrWithAnUnknownSeverityRaisesAWarning(t *testing.T) { // 1278/AC2
|
||||
for _, extra := range []string{``, `,"severity":""`, `,"severity":"critical"`, `,"severity":"warning"`} {
|
||||
line := `{"id":"novox/a#1","state":"unanswered","for":"3m0s","bound":"2m0s","h2":"close","says":"x"` + extra + `}`
|
||||
if got := severityRaisedFor(t, line); got != conditions.Warning {
|
||||
t.Errorf("the line %s raised %q", extra, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0419: a group's order, from the module graph and the manifests' change alone, the same on every
|
||||
// reading — what a module stands on before it, the controller before a manifest that needs it, the node-engine
|
||||
// before the controller, a verb's server before its caller — and no `after:` line read.
|
||||
func TestAGroupIsOrderedByTheGraphAndTheManifestsChange(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "cat", Repository: "novox/mesh-catalog"},
|
||||
{ID: "ctl", Repository: "novox/mesh-controller"},
|
||||
{ID: "host", Repository: "novox/mesh-host"},
|
||||
{ID: "base", Repository: "novox/base"},
|
||||
{ID: "app", Repository: "novox/app"},
|
||||
{ID: "lab", Repository: "novox/lab"},
|
||||
}
|
||||
reach := map[string]orderReach{
|
||||
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
|
||||
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
|
||||
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
|
||||
"base": {Moved: []string{"base"}},
|
||||
"app": {Moved: []string{"app"}},
|
||||
"lab": {Moved: []string{"lab"}},
|
||||
}
|
||||
edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeStandsOn},
|
||||
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
|
||||
changes := map[string]moduleChange{"gitea": {Manifest: true}, "mesh-controller": {Manifest: true},
|
||||
"mesh-host": {Manifest: true}, "lab": {Serves: []string{"seat.v"}}, "app": {Invokes: []string{"seat.v"}}}
|
||||
got := orderOf(members, reach, edges, changes)
|
||||
if len(got.Cycle) > 0 {
|
||||
t.Fatalf("a cycle where there is none: %v", got.Cycle)
|
||||
}
|
||||
// By repository among those with nothing before them: app waits for base and lab; gitea's catalogue for the controller.
|
||||
want := []string{"base", "lab", "app", "host", "ctl", "cat"}
|
||||
if !reflect.DeepEqual(got.Order, want) {
|
||||
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
|
||||
}
|
||||
why := map[string]string{}
|
||||
for _, p := range got.Pairs {
|
||||
why[p.Before+">"+p.After] = p.Why
|
||||
}
|
||||
for pair, reason := range map[string]string{"host>ctl": ruleEngineFirst, "ctl>cat": ruleVersionSkew,
|
||||
"ctl>host": "", "base>app": orderBuiltBy, "lab>app": ruleVerbBeforeCaller + ": seat.v"} {
|
||||
if why[pair] != reason {
|
||||
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
|
||||
}
|
||||
}
|
||||
// The same members in another order read the same.
|
||||
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
|
||||
if again := orderOf(shuffled, reach, edges, changes); !reflect.DeepEqual(again.Order, want) {
|
||||
t.Fatalf("another reading ordered %v", again.Order)
|
||||
}
|
||||
}
|
||||
|
||||
// A cycle is the one refusal, and it names every module and edge. A description's `after:` line changes nothing.
|
||||
func TestAGroupInACycleIsNamedAndNothingElseIsRefused(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "x", Repository: "novox/x", Number: 7},
|
||||
{ID: "y", Repository: "novox/y", Number: 9},
|
||||
}
|
||||
reach := map[string]orderReach{"x": {Moved: []string{"mx"}}, "y": {Moved: []string{"my"}}}
|
||||
changes := map[string]moduleChange{
|
||||
"mx": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}},
|
||||
"my": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}},
|
||||
}
|
||||
got := orderOf(members, reach, nil, changes)
|
||||
if !reflect.DeepEqual(got.Cycle, []string{"x", "y"}) || len(got.Order) > 0 {
|
||||
t.Fatalf("a cycle was ordered: %v, cycle %v", got.Order, got.Cycle)
|
||||
}
|
||||
for _, w := range []string{"mx", "my", "s.a", "s.b"} {
|
||||
if !strings.Contains(got.CycleSaid, w) {
|
||||
t.Errorf("the cycle does not say %q: %s", w, got.CycleSaid)
|
||||
}
|
||||
}
|
||||
// Without the edges the same members are ordered by repository.
|
||||
got = orderOf(members, reach, nil, nil)
|
||||
if !reflect.DeepEqual(got.Order, []string{"x", "y"}) || len(got.Cycle) > 0 || len(got.Pairs) > 0 {
|
||||
t.Fatalf("members with no edge: %v %v %+v", got.Order, got.Cycle, got.Pairs)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
|
||||
// record, and never one that moves a module on the controller's own path.
|
||||
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
|
||||
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
||||
unassigned := holder
|
||||
unassigned.On = nil
|
||||
for _, c := range []struct {
|
||||
entries []inventory.Entry
|
||||
moved []string
|
||||
waits bool
|
||||
}{
|
||||
{nil, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
|
||||
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"nats"}, false},
|
||||
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
|
||||
} {
|
||||
d := awaitsFor(c.entries, c.moved)
|
||||
if (d != nil) != c.waits {
|
||||
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
|
||||
c.moved, d != nil, c.waits)
|
||||
}
|
||||
if d != nil && d.Awaits != catalogue.DeliverySeat {
|
||||
t.Errorf("waits for %q", d.Awaits)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
|
||||
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
|
||||
// hand when the owner is down, and the owner's stop ends one as stopped.
|
||||
|
||||
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
|
||||
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
|
||||
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
|
||||
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
|
||||
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
|
||||
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
|
||||
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
|
||||
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
|
||||
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("deliver", map[string]any{}); err == nil {
|
||||
t.Error("deliver without a walk was composed")
|
||||
}
|
||||
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
|
||||
t.Error("a stop without why was composed")
|
||||
}
|
||||
}
|
||||
|
||||
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
|
||||
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else
|
||||
// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a
|
||||
// module built on one of those moves as well.
|
||||
func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) {
|
||||
entry := func(name string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: name}}
|
||||
}
|
||||
entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")}
|
||||
against := map[string][]string{
|
||||
"shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||
"shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"},
|
||||
"postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||
"unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"},
|
||||
}
|
||||
got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against)
|
||||
var names []string
|
||||
for _, e := range got {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true}
|
||||
if len(names) != len(want) {
|
||||
t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names)
|
||||
}
|
||||
for _, n := range names {
|
||||
if !want[n] {
|
||||
t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names)
|
||||
}
|
||||
}
|
||||
// Nothing moved: nothing follows.
|
||||
if more := dependentsOf(nil, entries, against); len(more) != 0 {
|
||||
t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more))
|
||||
}
|
||||
}
|
||||
@@ -79,7 +79,7 @@ var probeRegistry = []probe{
|
||||
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
|
||||
"assigned to a machine is left out of its declaration unsaid",
|
||||
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
|
||||
Raises: []string{kindLeftOut, kindNeedsOperator}, Phase: 1,
|
||||
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
|
||||
run: probeDeclarations},
|
||||
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
|
||||
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
|
||||
@@ -115,11 +115,26 @@ var probeRegistry = []probe{
|
||||
kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting},
|
||||
Phase: 2, run: probeData,
|
||||
Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}},
|
||||
// The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by
|
||||
// the controller, and H2 works it through the owner's `close`.
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a
|
||||
// node's presence, which the silent check takes.
|
||||
{ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " +
|
||||
"not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156",
|
||||
Kind: kindStuck, Phase: 3, run: probeStuck},
|
||||
// A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the
|
||||
// operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8).
|
||||
{ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " +
|
||||
"minutes for mesh-delivery's announcement of the operator's Approve of its head", From: "ADR 0432, issue 1146",
|
||||
Kind: kindFirstBuildUnasked, Phase: 3, run: probeFirstBuilds},
|
||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
||||
"its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304",
|
||||
Kind: kindOverBudget, Phase: 3, run: probeBudgets},
|
||||
// A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed
|
||||
// connections, which the bus's own module reads.
|
||||
{ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " +
|
||||
@@ -153,13 +168,30 @@ var probeRegistry = []probe{
|
||||
{ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " +
|
||||
"ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth},
|
||||
{ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " +
|
||||
"build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth},
|
||||
{ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus",
|
||||
From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth},
|
||||
{ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " +
|
||||
"it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4,
|
||||
run: probeBusHealth},
|
||||
// The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its
|
||||
// condition until a newer build passes; each rollback a witness stands by is said.
|
||||
{ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " +
|
||||
"a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack,
|
||||
Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates},
|
||||
// The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus.
|
||||
{ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " +
|
||||
"health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8",
|
||||
Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep},
|
||||
// A delivery plan held for a backup that failed or stayed silent (ADR 0404 decision 4): one condition per plan.
|
||||
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
|
||||
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
|
||||
run: probeBackupHeld},
|
||||
// A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it.
|
||||
{ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " +
|
||||
"bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1,
|
||||
run: probeSendKeyRefused},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said
|
||||
// it reads one, and to no other: an older node-engine refuses a key it does not know, whole.
|
||||
|
||||
// bodiesDelivery records each send as the mesh does, and keeps the bodies.
|
||||
type bodiesDelivery struct {
|
||||
recordedDelivery
|
||||
bodies map[string][]byte
|
||||
}
|
||||
|
||||
func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
b.bodies[s.node] = body
|
||||
return b.recordedDelivery.declare(ctx, s, body)
|
||||
}
|
||||
|
||||
func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
epoch := uint64(57)
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return epoch, nil }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
|
||||
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open), d, "", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
carried := func(node string) (epoch float64, has bool) {
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal(d.bodies[node], &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
epoch, has = envelope["epoch"].(float64)
|
||||
return epoch, has
|
||||
}
|
||||
if e, has := carried("anchor"); !has || e != 57 {
|
||||
t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"])
|
||||
}
|
||||
if _, has := carried("laptop"); has {
|
||||
t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"])
|
||||
}
|
||||
|
||||
// A new holder of the lease is not a change of the machine: neither reads as behind.
|
||||
epoch = 58
|
||||
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
sent, err := inv.Outstanding(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if would[node] != sent {
|
||||
t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A process that may not act composes nothing and sends nothing: its number is not taken.
|
||||
func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := epochForActs
|
||||
epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { epochForActs = was })
|
||||
d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}}
|
||||
refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open), d, "", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") {
|
||||
t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused)
|
||||
}
|
||||
anchor, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||
if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 {
|
||||
t.Fatalf("a controller without the lease took sequence %d", seq)
|
||||
}
|
||||
|
||||
// And at the send itself: the gate every declaration passes.
|
||||
gate := link.ActingGate
|
||||
link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") }
|
||||
t.Cleanup(func() { link.ActingGate = gate })
|
||||
if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil ||
|
||||
!strings.Contains(err.Error(), "lost the lease") {
|
||||
t.Fatalf("a declaration was let through the gate: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
snapshot "git.novox.be/novox/mesh-sdk/go/facts"
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -313,6 +315,13 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
m.Assigned = assigned
|
||||
sent, known, err := inv.SentBuilds(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
}
|
||||
if known && slices.Contains(assigned, broker.RuntimeModule) {
|
||||
m.NodeTools = sent[broker.RuntimeModule]
|
||||
}
|
||||
pins, err := inv.PinsFor(ctx, n.Name)
|
||||
if err != nil {
|
||||
return snapshot.Facts{}, err
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450).
|
||||
//
|
||||
// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy
|
||||
// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the
|
||||
// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never
|
||||
// listed on the condition that is open.
|
||||
|
||||
// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check.
|
||||
var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"}
|
||||
|
||||
// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why.
|
||||
func judgedRefused() inventory.ResourceHealth {
|
||||
r := waitingDatabaseFor(refusedWait)
|
||||
r.State, r.Waits = link.StateUnhealthy, nil
|
||||
r.Reason = "says it waits for the secret certificate, which db does not declare"
|
||||
return r
|
||||
}
|
||||
|
||||
// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list
|
||||
// it at the provider's unhealthy condition, made urgent by who waits on it.
|
||||
func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) {
|
||||
k, _ := withConditionsInMemory(t)
|
||||
stored := waitingDatabaseFor(refusedWait)
|
||||
open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()},
|
||||
map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}},
|
||||
shopFailingBeside(stored))
|
||||
provider := conditionOf(open, moduleUnhealthyKey("db", "anchor"))
|
||||
if len(open) != 1 || provider == nil {
|
||||
t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+
|
||||
"unhealthy alone", openKeysOf(open))
|
||||
}
|
||||
if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") {
|
||||
t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said)
|
||||
}
|
||||
if provider.Severity != conditions.Urgent {
|
||||
t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens:
|
||||
// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when
|
||||
// open, is said as unhealthy with who waits on it.
|
||||
func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) {
|
||||
given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase)
|
||||
given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)}
|
||||
by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")})
|
||||
if !held || by.provider != theDatabase || len(by.waits) > 0 {
|
||||
t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+
|
||||
"held under db on anchor as unhealthy, no waits", held, by.provider, by.waits)
|
||||
}
|
||||
if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered {
|
||||
t.Fatalf("a provider whose wait failed its check is said as waiting for another part")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// firstBuilds asks the first build of every module a merge adds to a repository no registered module is built from, when
|
||||
// the merge was announced as made on the operator's Approve of that head (novox/hq ADR 0432, issue 1146): the modules'
|
||||
// directories are the ones the announcement says hold a module.json at the merge commit, as touchedBy reads them. The
|
||||
// merge is kept, so the build's outcome is judged by the rule that registers it (mayRegisterFrom, approvedMove).
|
||||
//
|
||||
// **Complete in either order, with no timer** (ADR 0426): when mesh-delivery's word is kept first, it is found here; when
|
||||
// the forge's merge is heard first, the merge is kept as a first merge waiting for the word, and keepingDeliveryMerges
|
||||
// runs this again when the word is kept (resumeFirstBuilds). A word that says no (a Decline, another head) asks nothing and
|
||||
// says why. Answers whether the merge was dealt with here.
|
||||
func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, now time.Time) (bool, error) {
|
||||
inv := f.open.inventory
|
||||
repository := m.Owner + "/" + m.Repo
|
||||
added, why, err := firstBuildsOf(ctx, inv, m, entries, read)
|
||||
switch {
|
||||
case errors.Is(err, errFirstNotApproved):
|
||||
fmt.Printf("%s merged into %s (%.8s) adds %s, and no build is asked: %v\n", repository, m.Base, m.Commit,
|
||||
strings.Join(addedDirs(m, entries, read), ", "), err)
|
||||
return true, nil
|
||||
case errors.Is(err, errNotAnnounced):
|
||||
// Kept, waiting for mesh-delivery's word: a restart keeps it, and the word's arrival asks the builds.
|
||||
if kerr := keepFirstMerge(ctx, inv, m, now, true); kerr != nil {
|
||||
return true, notNow(kerr)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) adds %s; it is kept, and the first build is asked when mesh-delivery "+
|
||||
"announces the merge on the operator's Approve\n", repository, m.Base, m.Commit,
|
||||
strings.Join(addedDirs(m, entries, read), ", "))
|
||||
return true, nil
|
||||
case err != nil:
|
||||
return true, notNow(err)
|
||||
case len(added) == 0:
|
||||
return false, nil
|
||||
}
|
||||
if err := askFirstBuilds(ctx, f.open, m, now, added, why); err != nil {
|
||||
return true, notNow(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keepFirstMerge keeps a first merge as the controller heard it, waiting (no plan answers it).
|
||||
func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, waitingForWord bool) error {
|
||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||
if err != nil {
|
||||
merged = now
|
||||
}
|
||||
event, err := json.Marshal(keptMerge{SourceMoved: m, FirstBuild: waitingForWord})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = inv.AddMerge(ctx, inventory.BatchedMerge{Repository: m.Owner + "/" + m.Repo, Branch: m.Base, Commit: m.Commit,
|
||||
Merged: merged.UTC(), Heard: now, Event: event})
|
||||
return err
|
||||
}
|
||||
|
||||
// askFirstBuilds keeps the merge as answered and registers its modules from the builder's done keys (novox/hq ADR 0501
|
||||
// §3, §6): nothing asks their first builds. The merge kept is the registration's warrant (approvedMove reads it by the
|
||||
// commit the indexer's trunk value names); a module whose key is not done yet is registered when the key's write wakes
|
||||
// the take-in (new_modules.go).
|
||||
func askFirstBuilds(ctx context.Context, open *stores, m link.SourceMoved, now time.Time, added []string,
|
||||
why string) error {
|
||||
inv := open.inventory
|
||||
repository := m.Owner + "/" + m.Repo
|
||||
if err := keepFirstMerge(ctx, inv, m, now, false); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so %s "+
|
||||
"are registered from the builder's done keys, asking no build\n", repository, m.Base, m.Commit, why,
|
||||
strings.Join(added, ", "))
|
||||
// Answered: it no longer waits, so the word's arrival again does nothing twice.
|
||||
if err := inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false); err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := newModulesOf(ctx, repository)
|
||||
switch {
|
||||
case err != nil:
|
||||
// Said, not failed: the merge is answered, and the look at every module when a controller starts, or the next
|
||||
// write of the builder or the indexer for one of them, registers them.
|
||||
fmt.Printf(" the indexer's modules of %s cannot be read now (%v): they are registered when the builder's next "+
|
||||
"write of one wakes the take-in\n", repository, err)
|
||||
case len(modules) > 0:
|
||||
inBackground(func() { takeNewModules(context.WithoutCancel(ctx), open, modules...) })
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull
|
||||
// request's head: the other order of the same two events.
|
||||
func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error {
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
inv := f.open.inventory
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || len(k.Held) > 0 || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||
k.HeadSHA != d.HeadSHA {
|
||||
continue
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read)
|
||||
switch {
|
||||
case errors.Is(err, errFirstNotApproved):
|
||||
fmt.Printf("%s/%s merged into %s (%.8s) was waiting for mesh-delivery's word, and no build is asked: %v\n",
|
||||
k.Owner, k.Repo, k.Base, k.Commit, err)
|
||||
if aerr := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); aerr != nil {
|
||||
return notNow(aerr)
|
||||
}
|
||||
case err != nil:
|
||||
return notNow(err)
|
||||
case len(added) > 0:
|
||||
if err := askFirstBuilds(ctx, f.open, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFirstNotApproved says a merge into a repository the mesh builds nothing from is not one the operator approved.
|
||||
var errFirstNotApproved = errors.New("it is no merge made on the operator's Approve of its head, so a hand `build` " +
|
||||
"registers a module from a repository the mesh does not build yet")
|
||||
|
||||
// addedDirs are the module directories the merge adds to a repository no registered module is built from, none where
|
||||
// the merge is not into the trunk or some module is built from the repository already.
|
||||
func addedDirs(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []string {
|
||||
if m.Base != "main" {
|
||||
return nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && sameRepository(e.Source.Repository, m) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
_, added, _ := touchedBy(nil, entries, m, read)
|
||||
return added
|
||||
}
|
||||
|
||||
// firstBuildsOf is the directories whose first build a merge asks, and the warrant it was made on; none when the merge
|
||||
// is no first merge into a repository the mesh builds nothing from. errFirstNotApproved when it is and the word on record
|
||||
// is no Approve of its head; errNotAnnounced when mesh-delivery's word is not kept yet.
|
||||
func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) ([]string, string, error) {
|
||||
added := addedDirs(m, entries, read)
|
||||
if len(added) == 0 {
|
||||
return nil, "", nil
|
||||
}
|
||||
why, err := approvedAnnounced(ctx, inv, m.Owner, m.Repo, keptMerge{SourceMoved: m})
|
||||
switch {
|
||||
case err == nil:
|
||||
return added, why, nil
|
||||
case errors.Is(err, errWordNotAnApprove):
|
||||
return nil, "", fmt.Errorf("%w (%v)", errFirstNotApproved, err)
|
||||
}
|
||||
// Not announced yet, or the bus or the store could not be asked: the caller waits or tries again, and asks nothing.
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read,
|
||||
// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173). Read from a merge kept
|
||||
// before R4e, never written since: nothing asks a first build (ADR 0501 §6), and a controller that starts answers such a
|
||||
// merge and registers its modules from their keys (answerHeldMerges).
|
||||
type heldFirstBuild struct {
|
||||
Module string `json:"module"`
|
||||
Path string `json:"path"`
|
||||
Word string `json:"word"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
|
||||
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
|
||||
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
|
||||
|
||||
// refusedWord is the word a build's failure says the reader did not know; empty when it says none.
|
||||
func refusedWord(failed string) string {
|
||||
if m := unknownWord.FindStringSubmatch(failed); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
|
||||
const probeFirstBuildsID = "D18"
|
||||
|
||||
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
|
||||
const kindFirstBuildUnasked = "first-build-unasked"
|
||||
|
||||
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
|
||||
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
|
||||
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
|
||||
const firstBuildWaitBound = 10 * time.Minute
|
||||
|
||||
// firstBuildObservations are the conditions of the first merges waiting past the bound, one for each; a merge answered is
|
||||
// no longer listed as waiting, so its condition clears with the next look.
|
||||
func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
|
||||
continue
|
||||
}
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
for_ := now.Sub(w.Heard).Round(time.Minute)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge,
|
||||
ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), Kind: kindFirstBuildUnasked, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the merge %.8s of %s adds modules the mesh builds nothing from yet, and has waited %s for "+
|
||||
"mesh-delivery's announcement of the operator's Approve of its head: no build is asked", k.Commit, repository, for_),
|
||||
Said: fmt.Sprintf("%s at %s waits for mesh-delivery's word on pull request %d (head %.8s)", repository, k.Commit,
|
||||
k.Number, k.HeadSHA),
|
||||
Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository),
|
||||
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+
|
||||
"only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+
|
||||
"announcement has come in %s. The merge is kept, and the modules are registered from the builder's keys the moment "+
|
||||
"the announcement is kept.",
|
||||
k.Commit, repository, for_),
|
||||
Resolved: "mesh-delivery announces the merge, or the operator registers the modules at the controller's terminal",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
|
||||
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
waiting, err := d.open.inventory.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
return append(firstBuildObservations(waiting, now), newModuleObservations(now)...), nil
|
||||
}
|
||||
|
||||
// answerHeldMerges answers every merge a controller of before held for a build-agent that reads its new modules'
|
||||
// manifests (issue 1173): nothing asks their first builds any more, and the look at every module when this controller
|
||||
// starts registers them from their keys (ADR 0501 §6, §7).
|
||||
func answerHeldMerges(ctx context.Context, inv *inventory.Inventory) {
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("the merges held for a newer build-agent could not be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
for _, w := range waiting {
|
||||
if k := announcedOf(w); len(k.Held) > 0 {
|
||||
if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil {
|
||||
fmt.Printf("the merge %.8s of %s held for a newer build-agent could not be answered: %v\n", w.Commit,
|
||||
w.Repository, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("the merge %.8s of %s was held for a newer build-agent; its modules are registered from the "+
|
||||
"builder's keys instead\n", w.Commit, w.Repository)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1146/AC7 on keys (ADR 0501 §3, §6; issue 1851, A5), whole against a real bus and the store, in both orders of the two
|
||||
// events: mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's
|
||||
// merge, heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the
|
||||
// controller consumes with the router's record of the ask in its bucket, come in either order; each way the module the
|
||||
// merge commit holds is registered from the builder's done key of it, and nothing asks a build. Without the Approve in
|
||||
// the record it is not registered. Store-backed: skips without one.
|
||||
func TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
wordFirst bool
|
||||
approved bool // the router's record holds the operator's Approve of the merged head
|
||||
wantRegister bool
|
||||
}{
|
||||
{what: "the forge's merge before mesh-delivery's word", approved: true, wantRegister: true},
|
||||
{what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegister: true},
|
||||
{what: "a word whose record holds no Approve", wordFirst: true},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
noBuildAsked(t)
|
||||
b := aBuilderOnTheBus(t)
|
||||
js := b.js
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
if c.approved {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The indexer's value and the builder's done key of the module the merge commit holds.
|
||||
fp := fpOf('9')
|
||||
manifest := b.indexed(t, "mesh-delivery-view", "1", "novox/mesh-delivery", ".", commit, fp)
|
||||
b.built(t, "mesh-delivery-view", "1", fp, manifest, sdkdelivery.BuildDone)
|
||||
|
||||
following := following{open: open}
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims, kept: following.resumeFirstBuilds}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.AssertServingConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
word := func() {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the word being kept", func() bool {
|
||||
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, head)
|
||||
return err == nil && kept
|
||||
})
|
||||
time.Sleep(500 * time.Millisecond) // the keeper's hook runs as the word is kept
|
||||
}
|
||||
forge := func() {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := following.SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if c.wordFirst {
|
||||
word()
|
||||
forge()
|
||||
} else {
|
||||
forge()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known {
|
||||
t.Fatal("1146/AC7: registered before mesh-delivery's word was kept")
|
||||
}
|
||||
word()
|
||||
}
|
||||
if !c.wantRegister {
|
||||
time.Sleep(time.Second)
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known {
|
||||
t.Fatal("1146/AC7: a merge whose record holds no Approve registered its module")
|
||||
}
|
||||
return
|
||||
}
|
||||
becomes(t, 10*time.Second, "the module registered", func() bool {
|
||||
_, known := inventoryHolds(t, inv, "mesh-delivery-view")
|
||||
return known
|
||||
})
|
||||
src, err := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||
if err != nil || src.Repository != "novox/mesh-delivery" {
|
||||
t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) {
|
||||
t.Helper()
|
||||
entries, err := inv.Catalogued(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == module {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Entry{}, false
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1146/AC6: the first merge into a repository no registered module is built from asks the first build of every module
|
||||
// the merge commit holds when it was made on the operator's Approve of that head, and asks nothing without it.
|
||||
func TestAFirstMergeIntoANewRepositoryAsksItsModulesBuildsOnlyOnTheApprove(t *testing.T) {
|
||||
const head = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
now := time.Date(2026, 10, 19, 0, 6, 0, 0, time.UTC)
|
||||
mergedAt := now.Add(-time.Hour).Format(time.RFC3339Nano)
|
||||
heard := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: "0177ea7a",
|
||||
Number: 1, MergedAt: mergedAt, ModuleDirsSaid: true, ModuleDirs: []string{"watchman", "view"},
|
||||
Paths: []string{"view/module.json", "watchman/module.json", "README.md"}}
|
||||
built := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git"}}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge, *link.SourceMoved, *[]inventory.Entry)
|
||||
unkept bool
|
||||
dirs []string
|
||||
refused bool
|
||||
waits bool
|
||||
}{
|
||||
{what: "the Approve of the merged head", dirs: []string{"view", "watchman"}},
|
||||
{what: "a Decline", refused: true, change: func(a *anApprovedMerge, _ *link.SourceMoved, _ *[]inventory.Entry) {
|
||||
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
||||
}},
|
||||
{what: "no word of mesh-delivery yet, however long ago the merge was made", unkept: true, waits: true},
|
||||
{what: "a module already built from the repository", change: func(_ *anApprovedMerge, _ *link.SourceMoved, e *[]inventory.Entry) {
|
||||
*e = append(*e, inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}})
|
||||
}},
|
||||
{what: "a merge into another branch", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { m.Base = "feat" }},
|
||||
{what: "a merge that holds no module.json", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) {
|
||||
m.ModuleDirs, m.Paths = nil, []string{"README.md"}
|
||||
}},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
a.merge.Number = 1
|
||||
m, entries := heard, []inventory.Entry{built}
|
||||
if c.change != nil {
|
||||
c.change(&a, &m, &entries)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
dirs, _, err := firstBuildsOf(t.Context(), nil, m, entries, nil)
|
||||
switch {
|
||||
case c.waits && !errors.Is(err, errNotAnnounced):
|
||||
t.Fatalf("1146/AC6: %s: want a wait for the word, got %v", c.what, err)
|
||||
case c.refused && !errors.Is(err, errFirstNotApproved):
|
||||
t.Fatalf("1146/AC6: %s: want the Approve missing, got %v (dirs %v)", c.what, err, dirs)
|
||||
case !c.refused && !c.waits && err != nil:
|
||||
t.Fatalf("1146/AC6: %s: %v", c.what, err)
|
||||
case !slices.Equal(dirs, c.dirs):
|
||||
t.Fatalf("1146/AC6: %s: first builds asked of %v, want %v", c.what, dirs, c.dirs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1146/AC8: a first merge waiting past the bound for mesh-delivery's word is said, naming the repository, the commit and
|
||||
// the missing word; one inside the bound, one that is no first merge, and one answered (no longer waiting) are not.
|
||||
func TestAFirstMergeWaitingPastItsBoundIsSaidAndClearsWhenAnswered(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC)
|
||||
waiting := func(commit string, first bool, heard time.Time) inventory.BatchedMerge {
|
||||
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
|
||||
Commit: commit, Number: 1, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: first})
|
||||
return inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, Heard: heard, Event: event}
|
||||
}
|
||||
got := firstBuildObservations([]inventory.BatchedMerge{
|
||||
waiting("0177ea7a00", true, now.Add(-firstBuildWaitBound-time.Minute)),
|
||||
waiting("1111111100", true, now.Add(-time.Minute)),
|
||||
waiting("2222222200", false, now.Add(-time.Hour)),
|
||||
}, now)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("1146/AC8: %d conditions raised, want the one merge past the bound: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != kindFirstBuildUnasked || o.Scope != conditions.ScopeMerge || o.Severity != conditions.Warning ||
|
||||
!strings.Contains(o.Summary, "novox/mesh-delivery") || !strings.Contains(o.Summary, "0177ea7a") ||
|
||||
!strings.Contains(o.Summary, "announcement of the operator's Approve") {
|
||||
t.Fatalf("1146/AC8: the condition is %+v", o)
|
||||
}
|
||||
// Answered, the merge is no longer listed as waiting: nothing is observed, and the condition clears.
|
||||
if cleared := firstBuildObservations(nil, now); len(cleared) != 0 {
|
||||
t.Fatalf("1146/AC8: an answered merge still raises %+v", cleared)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the
|
||||
// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core,
|
||||
// leaf or none, so mesh-delivery can pass it on with the fix's delivery.
|
||||
func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
entry := func(name, path string) inventory.Entry {
|
||||
e := fromRepo(name, repo, path)
|
||||
e.Source.BuiltFrom = "old"
|
||||
e.On = []string{"anchor"}
|
||||
return e
|
||||
}
|
||||
gitea := entry("gitea", "modules/gitea")
|
||||
resolver := entry("dnsmasq", "modules/dnsmasq")
|
||||
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
||||
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
||||
delivery.Manifest.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||
webApp := entry("photos", "modules/photos")
|
||||
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
||||
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
||||
edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}}
|
||||
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
plan := func(paths ...string) link.ChangePlan {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths}
|
||||
return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy)
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
paths []string
|
||||
want string
|
||||
}{
|
||||
{"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf},
|
||||
{"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore},
|
||||
{"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
||||
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-host/module.json"}, inventory.ClassCore},
|
||||
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
} {
|
||||
if got := plan(c.paths...).Class; got != c.want {
|
||||
t.Errorf("%s: class %q, want %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
passes bool
|
||||
}{
|
||||
{"found a day before the send", -24 * time.Hour, true},
|
||||
{"found by this send", time.Hour, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
||||
return map[string]bool{"anchor": true, "laptop": true}, nil
|
||||
}
|
||||
backlogFacts := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f, err := backlogFacts(ctx, open, component)
|
||||
// The used-as-found condition, raised after the send (its second statement): its own kind, never a
|
||||
// fault the gate reads as the build's.
|
||||
f.judged, f.openErr = true, nil
|
||||
f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound,
|
||||
Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"},
|
||||
Raised: time.Now()})
|
||||
f.health = map[string]inventory.NodeHealth{}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{
|
||||
{Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy},
|
||||
foundDirectory("app", time.Now().Add(c.found)),
|
||||
{Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}}
|
||||
}
|
||||
return f, err
|
||||
}
|
||||
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
|
||||
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
|
||||
wasWindow := gateWitnessWindow
|
||||
gateWitnessWindow = 0
|
||||
t.Cleanup(func() { gateWitnessWindow = wasWindow })
|
||||
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
|
||||
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
advancePlans(ctx, b.open)
|
||||
if p := b.release(t); p.State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := b.release(t)
|
||||
v, found, err := inv.GateOf(ctx, "build-app-c2")
|
||||
if c.passes {
|
||||
if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v)
|
||||
}
|
||||
if !strings.Contains(v.Why+p.Note, "hand it over") {
|
||||
t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note)
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.State == inventory.PlanDone {
|
||||
t.Fatalf("a directory this send found let the walk through: %s", p.Note)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
open, _ := k.Open(ctx)
|
||||
var got *conditions.Condition
|
||||
for i, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
got = &open[i]
|
||||
}
|
||||
if c.Kind == kindModuleUnhealthy {
|
||||
t.Fatalf("raised as a fault: %+v", c)
|
||||
}
|
||||
}
|
||||
if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent ||
|
||||
!strings.Contains(got.Summary, "notes.data") {
|
||||
t.Fatalf("the condition: %+v", got)
|
||||
}
|
||||
// Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait.
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent {
|
||||
t.Fatal("a directory used as found became urgent")
|
||||
}
|
||||
}
|
||||
if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, _ = k.Open(ctx)
|
||||
for _, c := range open {
|
||||
if c.Key == usedAsFoundKey("notes", "laptop") {
|
||||
t.Fatal("not cleared once handed over")
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user