novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: novox/mesh-delivery:main
Branches Tags
novox/mesh-delivery:main novox/mesh-delivery:feat/try-speed novox/mesh-delivery:fix/1921-every-failure novox/mesh-delivery:feat/r4g-finish-composer novox/mesh-delivery:fix/1901 novox/mesh-delivery:fix/r4e-r2 novox/mesh-delivery:fix/r4e-k1 novox/mesh-delivery:feat/r4e-builder novox/mesh-delivery:feat/r4e-checker novox/mesh-delivery:feat/r4e-view novox/mesh-delivery:feat/r4c5-checker
..
pull from: novox/mesh-delivery:fix/1921-every-failure
Branches Tags
novox/mesh-delivery:feat/try-speed novox/mesh-delivery:main novox/mesh-delivery:fix/1921-every-failure novox/mesh-delivery:feat/r4g-finish-composer novox/mesh-delivery:fix/1901 novox/mesh-delivery:fix/r4e-r2 novox/mesh-delivery:fix/r4e-k1 novox/mesh-delivery:feat/r4e-builder novox/mesh-delivery:feat/r4e-checker novox/mesh-delivery:feat/r4e-view novox/mesh-delivery:feat/r4c5-checker
1 Commits
main .. fix/1921-every-failure
Author SHA1 Message Date
jschoubben 6fe7b7dd6c Name every failed test in a check's verdict and a try's failed list (issue 1921) 
The checker stopped reading a level's test results at the first that did
not pass and kept only its first failure, so a check named one fault per
round. It now reads and counts every result and writes every failed test
with its first line into the verdict's Failures and the try's failed
list (at most 50, then how many more); Failed keeps the first one's
words whole (issue 1901). Pins mesh-sdk fix/1921-every-failure.
 2026-10-24 23:44:46 +02:00
48 changed files with 661 additions and 3646 deletions
Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+19 -477
View File
@@ -5,7 +5,6 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
@@ -18,12 +17,9 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
"mesh-delivery-assigner/internal/assign"
)
// bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events
@@ -77,10 +73,6 @@ func startBed(t *testing.T, leaveOut string) *bed {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
t.Fatal(err)
}
// The operator-channel seat's stream of conditions, which the assigner says its conditions on.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}}); err != nil {
t.Fatal(err)
}
// The Go build seat's work queue, as the broker derives it from the seat's accepted kinds (delivery.AskStream).
seat, _ := delivery.AskStream(delivery.AskGo)
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: seat, Subjects: []string{"mesh.seat.node-build-go.accept.>"},
@@ -93,14 +85,6 @@ func startBed(t *testing.T, leaveOut string) *bed {
// start runs the program as the node does, through run with its environment and a broker file; it answers the
// channel run's error arrives on.
func (b *bed) start(t *testing.T) <-chan error {
t.Helper()
done, _ := b.startStoppable(t)
return done
}
// startStoppable is start, and answers the function that stops this one program (a replaced assigner is stopped, then
// another started).
func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
t.Helper()
broker := filepath.Join(b.dir, "broker")
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"})
@@ -110,19 +94,15 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
env := map[string]string{"MESH_BROKER_FILE": broker}
ctx, stop := context.WithCancel(b.ctx)
done := make(chan error, 1)
go func() {
done <- run(ctx, func(k string) string { return env[k] }, io.Discard)
close(done) // a program that ended on its own (a refused start) is not waited for again at the cleanup
}()
stopped := func() {
go func() { done <- run(ctx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() {
stop()
select {
case <-done:
case <-time.After(10 * time.Second):
}
}
t.Cleanup(stopped)
return done, stopped
})
return done
}
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
@@ -179,27 +159,6 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
}
}
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
// move to it, in mesh-controller_composed.
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
t.Helper()
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
t.Fatal(err)
}
digest := "sha256:" + strings.Repeat(string(d), 64)
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
if judgement != delivery.JudgedNeither {
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
}
v, err := delivery.EncodeComposedDigest(cd)
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
return digest
}
// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest.
func (b *bed) manifest(t *testing.T, module, body string) string {
t.Helper()
@@ -281,10 +240,10 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte {
return nil
}
// 1297/B4 on the bus, 1906/D5: the controller sends ace and g14 builds while the canary ace does not run F healthy yet.
// Started whole, the assigner wants F on the canary, holds g14 back with no want, and writes no difference at all: there
// is no send to compare with (ADR 0539 §8).
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
// 1297/B4, B5 on the bus: the controller sent F to g14 while the canary ace does not run F healthy yet. Started whole,
// the assigner wants F on the canary, writes the difference on g14 explained by rule 3, and on ace one unexplained (the
// controller sends ace the older fingerprint while ace wants F).
func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
@@ -299,8 +258,6 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
gkey, _ := delivery.ControllerGrantKey("web", fNew)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
b.compose(t, "g14", "web", fNew, 'c', delivery.JudgedNeither)
for _, node := range []string{"ace", "g14"} {
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, node, on)
@@ -309,6 +266,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
}
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-8"), []byte(`{"id":"b-8","repository":"novox/web","source-fingerprint":"`+fOld+`"}`))
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-9"), []byte(`{"id":"b-9","repository":"novox/web","source-fingerprint":"`+fNew+`"}`))
b.start(t)
wkey, _ := delivery.WantKey("ace", "web")
@@ -316,9 +275,15 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if err != nil || want.FP != fNew {
t.Fatalf("the canary ace's want = %+v, %v", want, err)
}
time.Sleep(500 * time.Millisecond)
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
t.Errorf("differences were written: %v", keys)
dkey, _ := delivery.DifferenceKey("g14", "web")
d, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, dkey))
if err != nil || d.Rule != delivery.RuleCanary || d.Sent != fNew {
t.Fatalf("g14's difference = %+v, %v; want rule 3", d, err)
}
akey, _ := delivery.DifferenceKey("ace", "web")
a, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, akey))
if err != nil || !a.Unexplained || a.Want != fNew || a.Sent != fOld {
t.Fatalf("ace's difference = %+v, %v; want it unexplained", a, err)
}
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
@@ -350,426 +315,3 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
t.Fatalf("the ask = %+v, %v", a, err)
}
}
// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key.
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
t.Helper()
stream, err := b.js.Stream(b.ctx, "CONDITIONS")
if err != nil {
t.Fatal(err)
}
out := map[string]asks.Condition{}
cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy})
if err != nil {
t.Fatal(err)
}
for {
batch, err := cons.FetchNoWait(100)
if err != nil {
t.Fatal(err)
}
n := 0
for m := range batch.Messages() {
n++
c, err := asks.DecodeCondition(m.Data())
if err != nil {
t.Fatal(err)
}
out[c.Key] = c
}
if n == 0 {
return out
}
}
}
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node
// and the check's words.
func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64)
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "mail", mod)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.mail", failed)
wantKey, _ := delivery.WantKey("ace", "mail")
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
b.start(t)
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
t.Fatalf("the restore ask = %+v, %v", a, err)
}
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew {
t.Errorf("the data module was set back: want = %+v, %v", w, err)
}
key := ""
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
for k, c := range b.conditionsSaid(t) {
if c.State == asks.StateRaised && c.Subject.ID == "mail" {
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
}
key = k
}
}
}
if key == "" {
t.Fatal("no condition for the held fingerprint was said")
}
}
// 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants
// of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost
// and nothing written again.
func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
b := startBed(t, "")
const modules = 40
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
names := make([]string, modules)
for i := range names {
name := fmt.Sprintf("mod%02d", i)
names[i] = name
fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name))
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
if err != nil {
t.Fatal(err)
}
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
}
count := func() int {
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
if err != nil {
return 0
}
return len(keys)
}
_, stop := b.startStoppable(t)
for deadline := time.Now().Add(20 * time.Second); count() == 0 && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
}
stop() // replaced, wherever its pass had got to
b.start(t)
for deadline := time.Now().Add(30 * time.Second); count() < modules && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
}
time.Sleep(time.Second) // and no pass of the second writes any again
for _, name := range names {
key, _ := delivery.WantKey("ace", name)
history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key)
if err != nil {
t.Errorf("the want of %s was never written: %v", name, err)
continue
}
if len(history) != 1 {
t.Errorf("the want of %s was written %d times, want once", name, len(history))
}
}
}
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
func TestAWantWaitsForItsComposition(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, "web", mod)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey("web", fNew)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey("web", fNew)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
b.start(t)
wkey, _ := delivery.WantKey("ace", "web")
time.Sleep(time.Second)
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
t.Fatalf("a want was written before the composition existed: %s", e.Value())
}
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
}
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
break
}
}
}
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
}
// the node's own want
nkey, _ := delivery.WantNodeKey("ace")
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
}
ownKey, _ := delivery.ComposedNodeKey("ace")
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
t.Fatalf("the node's own want = %+v, %v", w, err)
}
}
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
t.Helper()
digest := b.manifest(t, name, manifest)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, name, mod)
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := delivery.BuilderBuildKey(name, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.ControllerGrantKey(name, fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
b.putJSON(t, delivery.PresenceBucket, "ace", on)
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
}
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
t.Helper()
key, _ := delivery.WantKey(node, module)
v, err := delivery.EncodeWant(w)
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AssignerWantsBucket, key, v)
}
// wantNow waits until the want of module on node satisfies ok and answers it.
func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.Want) bool) delivery.Want {
t.Helper()
key, _ := delivery.WantKey(node, module)
var last delivery.Want
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil {
if w, err := delivery.DecodeWant(e.Value()); err == nil {
last = w
if ok(w) {
return w
}
}
}
}
t.Fatalf("never saw: %s; the want is %+v", what, last)
return last
}
// answer says the teller wrote the operator's answer to the ask at key.
func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") }
// answerTo is answer to the ask with this id (the router's warrant names it).
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
t.Helper()
v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: askID + "@2026-10-24T12:00:00Z", Words: "the operator"})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.AnswersBucket, key, v)
}
func (b *bed) hasKey(bucket, key string) bool {
_, err := b.buckets[bucket].Get(b.ctx, key)
return err == nil
}
// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules,
// and clears the condition.
func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
b.start(t)
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld })
if w.Newest != fNew {
t.Fatalf("the set-back want = %+v", w)
}
hold, _ := delivery.HoldAskKey("web", fNew)
b.get(t, delivery.AssignerAsksBucket, hold)
b.answer(t, hold, "release")
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
t.Errorf("the want = %+v", w)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
cleared := false
for _, c := range b.conditionsSaid(t) {
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared)
}
if cleared {
return
}
}
t.Error("the condition of the held fingerprint was not cleared")
}
// askNow waits for the backup ask at key to be the one numbered asking and answers it.
func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk {
t.Helper()
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil {
if a, err := delivery.DecodeAssignerAsk(key, e.Value()); err == nil && a.Asking == asking {
return a
}
}
}
t.Fatalf("the ask at %s never became asking %d", key, asking)
return delivery.AssignerAsk{}
}
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
b := startBed(t, "")
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
b.start(t)
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
a := b.askNow(t, akey, 0)
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
t.Fatalf("the backup ask = %+v", a)
}
// an answer to another ask is no answer to this one
b.answerTo(t, akey, delivery.BackupSkip, "someone-else")
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
t.Fatalf("an answer to another ask was taken: %+v", w)
}
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
if w1.FP != fNew || w1.Composed != first {
t.Errorf("the want = %+v", w1)
}
// the node retries and the backup fails again: a new have value, asked again under the next number
again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, delivery.HaveBucket, "ace.web", again)
a2 := b.askNow(t, akey, 2)
if a2.Ask.ID == a.Ask.ID {
t.Fatalf("asked again under the same id")
}
b.answerTo(t, akey, delivery.BackupAgain, a2.Ask.ID)
w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain && w.Words != w1.Words })
if w2.Words == w1.Words {
t.Errorf("the second answer left the want as it was: %+v", w2)
}
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
a3 := b.askNow(t, akey, 3)
if !strings.Contains(a3.Ask.Explanation, second) {
t.Errorf("the ask for the new revision = %+v", a3)
}
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
t.Errorf("the old answer was taken for the new revision: %+v", w)
}
}
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
// neither is written at once.
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "web", fNew, recorded)
old := "sha256:" + strings.Repeat("a", 64)
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
b.start(t)
akey, _ := delivery.RecordAskKey("ace", "web", nw)
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
t.Fatalf("the record ask = %+v, %v", a, err)
}
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
t.Fatalf("the move was written before the yes: %+v", w)
}
b.answer(t, akey, assign.OptionYes)
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
// a new revision at the same fingerprint, judged to move data, waits as well
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
b.get(t, delivery.AssignerAsksBucket, rkey)
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
t.Fatalf("the revision was written before the yes: %+v", w)
}
// and a revision judged neither is written at once
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
}
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
b := startBed(t, "")
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
b.start(t)
time.Sleep(time.Second)
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
}
// the controller composes the bus at the fingerprint it runs with a new user list
key, _ := delivery.ComposedKey("ace", "bus", fOld)
free := "sha256:" + strings.Repeat("e", 64)
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
if w.FP != fOld {
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
}
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
t.Errorf("the user-list revision was asked")
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+3 -25
View File
@@ -21,7 +21,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -43,8 +42,6 @@ var readBuckets = []string{
provider.AssignmentsBucket,
delivery.BuildsBucket,
delivery.ControllerGrantsBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ManifestsBucket,
@@ -133,7 +130,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
return follow(ctx, js, kvs, logger)
}
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
@@ -141,8 +138,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
var watched = []string{
provider.AssignmentsBucket,
delivery.ControllerGrantsBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ModulesBucket,
@@ -156,7 +151,7 @@ var watched = []string{
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -220,7 +215,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
case <-settled:
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p := &passer{kvs: kvs, w: w, logf: logger.Printf}
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
wake()
for {
@@ -235,20 +230,3 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
}
}
}
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
// (ADR 0437 clause 9).
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
return func(c asks.Condition) error {
subject, err := asks.ConditionSubject(delivery.AssignerModule)
if err != nil {
return err
}
data, err := asks.EncodeCondition(c)
if err != nil {
return err
}
_, err = js.Publish(ctx, subject, data)
return err
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+60 -197
View File
@@ -3,10 +3,8 @@ package main
import (
"context"
"errors"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/sdkversion"
@@ -21,43 +19,76 @@ type passer struct {
w *world
logf func(string, ...any)
sdk string
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
// pass says a condition once and a restart says it again (the router keeps one per key).
say func(asks.Condition) error
said map[string]bool
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
answeredAt map[string]uint64
asks *composeAsks
asks *composeAsks
}
func (p *passer) pass(ctx context.Context) {
p.compositions(ctx)
p.wants(ctx)
p.nodeWants(ctx)
p.backups(ctx)
rules := p.wants(ctx)
p.differences(ctx, rules)
}
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
func (p *passer) nodeWants(ctx context.Context) {
written := p.w.read(delivery.AssignerWantsBucket)
for node, d := range p.w.nodeComposed() {
key, err := delivery.WantNodeKey(node)
// differences writes, for each node and module the assigner wants or the controller sends, the difference between the
// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that
// ended; its history keeps what it was.
func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) {
_, assignments := p.w.assigned()
wanted := p.w.wantsOf()
trunk := p.w.trunk()
pairs := map[[2]string]bool{}
for _, a := range assignments {
pairs[[2]string{a.Node, a.Module}] = true
}
for module, byNode := range wanted {
for node := range byNode {
pairs[[2]string{node, module}] = true
}
}
written := p.w.read(delivery.AssignerDifferencesBucket)
keep := map[string]bool{}
for pair := range pairs {
node, module := pair[0], pair[1]
key, err := delivery.DifferenceKey(node, module)
if err != nil {
continue
}
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
var want *delivery.Want
if w, ok := wanted[module][node]; ok {
want = &w
}
var sent assign.Sent
if a, ok := assignments[node+"."+module]; ok {
sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)}
}
d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node])
if !ok {
continue
}
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
keep[key] = true
if before, err := delivery.DecodeDifference(written[key]); err == nil {
before.V = 0
if delivery.SameDifference(before, d) {
continue
}
}
p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) })
}
for key := range written {
if keep[key] {
continue
}
if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil {
p.logf("the difference %s that ended could not be deleted: %v", key, err)
continue
}
p.w.put(delivery.AssignerDifferencesBucket, key, nil)
}
}
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
func (p *passer) wants(ctx context.Context) {
// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node
// back, for the comparison.
func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding {
get := kvGetter(p.kvs[delivery.ManifestsBucket])
assigned, _ := p.w.assigned()
trunk := p.w.trunk()
@@ -78,6 +109,7 @@ func (p *passer) wants(ctx context.Context) {
}
}
wanted := p.w.wantsOf()
rules := map[string]map[string]assign.Holding{}
names := map[string]bool{}
for name := range trunk {
names[name] = true
@@ -87,32 +119,18 @@ func (p *passer) wants(ctx context.Context) {
}
for name := range names {
fp := trunk[name].FP2
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name],
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
Restored: func(node string) bool {
key, err := delivery.RestoreAskKey(node, name, fp)
return err == nil && p.answeredWith(key, assign.OptionRestore)
},
Allowed: func(node, digest string) bool {
key, err := delivery.RecordAskKey(node, name, digest)
return err == nil && p.answeredWith(key, assign.OptionYes)
}}
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]}
if fp != "" {
held, _ := delivery.HeldKey(name, fp)
grant, _ := delivery.ControllerGrantKey(name, fp)
hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
if m.Released {
p.clearHeld(name, fp)
}
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant)
}
last := map[string]string{}
for _, node := range m.Nodes {
last[node] = p.w.lastHealthy(node, name, fp)
}
out := assign.Wants(m, nodes, wanted[name], last)
rules[name] = out.Rules
if out.Held != nil {
key, err := delivery.HeldKey(name, fp)
if err == nil {
@@ -122,13 +140,6 @@ func (p *passer) wants(ctx context.Context) {
}
if out.Held != nil || m.Held {
p.holdAsk(ctx, name, fp)
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
}
for _, node := range out.Restore {
p.restoreAsk(ctx, name, node, fp)
}
for _, need := range out.Record {
p.recordAsk(ctx, name, fp, need)
}
for node, want := range out.Wants {
key, err := delivery.WantKey(node, name)
@@ -140,6 +151,7 @@ func (p *passer) wants(ctx context.Context) {
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
}
}
return rules
}
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
@@ -218,152 +230,3 @@ func (p *passer) holdAsk(ctx context.Context, module, fp string) {
a := assign.HoldAsk(key, module, fp, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// heldHold is the hold of fp as written: the node whose health check failed and its words.
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
if fresh != nil {
return *fresh
}
key, err := delivery.HeldKey(module, fp)
if err != nil {
return delivery.Held{}
}
h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key])
return h
}
// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the
// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask.
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *delivery.Held, data bool) {
h := p.heldHold(module, fp, fresh)
c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data)
if p.say == nil || p.said[c.Key+".raised"] {
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err)
return
}
p.said[c.Key+".raised"] = true
}
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
key, err := delivery.RestoreAskKey(node, module, fp)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
return
}
words := p.heldHold(module, fp, nil).Words
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the
// operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19).
func (p *passer) answeredWith(key, option string) bool {
a, ok := p.w.answer(key)
return ok && a.Option == option
}
// clearHeld clears, once, the condition of a hold the operator released.
func (p *passer) clearHeld(module, fp string) {
c := assign.HeldCondition(module, fp, "", "", false, false)
if p.say == nil || p.said[c.Key+".cleared"] {
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
return
}
p.said[c.Key+".cleared"] = true
}
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
key, err := delivery.RecordAskKey(need.Node, module, need.Digest)
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
return
}
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
}
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
// it asks whether to back up again or go on without a backup, raises the condition naming node, module and the backup's words,
// and writes the answer into the node's want for the module, which the node agent reads from it and from nothing else.
//
// The ask is bound to the composition (it names the digest, and a new revision at the same fingerprint is asked again under a
// new number) and an answer counts only if the router's warrant names the id of the ask now at the key (the warrant's first part,
// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
// in a new have value after the answer was written.
func (p *passer) backups(ctx context.Context) {
wanted := p.w.wantsOf()
for key, value := range p.w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := delivery.DecodeHave(value)
if err != nil || have.Phase != delivery.PhaseHeld {
continue
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
}
akey, err := delivery.BackupAskKey(node, module, want.FP)
if err != nil {
continue
}
current, there := delivery.DecodeAssignerAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
if there != nil || assign.BackupAskDigest(current) != want.Composed {
after := -1
if there == nil {
after = current.Asking
}
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(after))
continue
}
answer, answered := p.w.answer(akey)
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != current.Ask.ID ||
(answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) {
continue // not answered yet: the ask stands
}
words := assign.BackupAnswerWords(answer.Option, current.Asking)
haveChanges := p.w.changed(delivery.HaveBucket, key)
if want.Backup != answer.Option || want.Words != words {
want.Backup, want.Words = answer.Option, words
if wkey, err := delivery.WantKey(node, module); err == nil {
w := want
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
p.answeredAt[current.Ask.ID] = haveChanges
}
continue
}
if answer.Option != delivery.BackupAgain {
continue
}
// The answer is in the want. The node retried and the backup failed again when the have took another value since.
if seen, known := p.answeredAt[current.Ask.ID]; !known {
p.answeredAt[current.Ask.ID] = haveChanges // after a restart: wait for the next failure
} else if haveChanges > seen {
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(current.Asking))
}
}
}
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
c := assign.BackupCondition(node, module, want.FP, words, true)
if p.say == nil || p.said[c.Key] {
return
}
if err := p.say(c); err != nil {
p.logf("the condition of the failed backup of %s on %s could not be said: %v", module, node, err)
return
}
p.said[c.Key] = true
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go
+23 -49
View File
@@ -44,8 +44,8 @@ type world struct {
manifests map[string]parsed
// healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to.
healthy map[string][]string
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
changes map[string]uint64
// sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one.
sent map[string]string
}
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
@@ -56,7 +56,7 @@ type parsed struct {
}
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}}
}
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
@@ -71,7 +71,6 @@ func (w *world) put(bucket, key string, value []byte) {
return
}
w.values[bucket][key] = value
w.changes[bucket+"|"+key]++
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
@@ -350,55 +349,30 @@ func (w *world) built(module, fp string) bool {
return err == nil && b.State == delivery.BuildDone
}
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
key, err := delivery.ComposedKey(node, module, fp)
// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record
// (mesh-controller_builds `<module>.build.<id>`) once and kept; empty when the record names none.
func (w *world) sentFP(ctx context.Context, get getter, module, build string) string {
if build == "" {
return ""
}
w.mu.Lock()
fp, ok := w.sent[build]
w.mu.Unlock()
if ok {
return fp
}
value, err := get(ctx, delivery.BuildKey(module, build))
if err != nil {
return delivery.ComposedDigest{}, false
return ""
}
var rec delivery.BuildRecord
if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) {
return ""
}
w.mu.Lock()
value, ok := w.values[delivery.ControllerComposedBucket][key]
w.sent[build] = rec.SourceFingerprint
w.mu.Unlock()
if !ok {
return delivery.ComposedDigest{}, false
}
d, err := delivery.DecodeComposedDigest(value)
return d, err == nil
}
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
out := map[string]delivery.ComposedDigest{}
for key, value := range w.read(delivery.ControllerComposedBucket) {
node, _, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || !isNode {
continue
}
if d, err := delivery.DecodeComposedDigest(value); err == nil {
out[node] = d
}
}
return out
}
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
func (w *world) answer(key string) (delivery.Answer, bool) {
w.mu.Lock()
value, ok := w.values[delivery.AnswersBucket][key]
w.mu.Unlock()
if !ok {
return delivery.Answer{}, false
}
a, err := delivery.DecodeAnswer(value)
return a, err == nil
}
// changed is how many values the key has taken in this process (see world.changes).
func (w *world) changed(bucket, key string) uint64 {
w.mu.Lock()
defer w.mu.Unlock()
return w.changes[bucket+"|"+key]
return rec.SourceFingerprint
}
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go
-14
View File
@@ -111,17 +111,3 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
}
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
t.Errorf("changed = %d, want 2", got)
}
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
}
modules/mesh-delivery-assigner/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-assigner/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 h1:ZEEeYibEPD1rngOAqbODzRY5rUStGC46OpyQwtNCYbs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-assigner/internal/assign/asks.go
+4 -149
View File
@@ -2,8 +2,6 @@ package assign
import (
"fmt"
"regexp"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
@@ -11,16 +9,17 @@ import (
)
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer
// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6.
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
"no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
"no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)),
Options: []asks.Option{
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
{ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
@@ -28,147 +27,3 @@ func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
About: key,
}}
}
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
const (
OptionRestore = "restore"
OptionLeaveStopped = "leave-stopped"
)
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
Options: []asks.Option{
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module stays as it is on the node",
About: key,
}}
}
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
// data, whose restore ask waits for the operator.
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
if needsYou {
c.Needs = "answer whether to restore " + module + " on " + node
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
} else {
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
}
return c
}
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
const (
OptionYes = "yes"
OptionNo = "no"
OptionBackupAgain = delivery.BackupAgain
OptionBackupSkip = delivery.BackupSkip
)
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
// after the answer yes. The way the move was judged is in the ask.
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
what := map[delivery.Judgement]string{
delivery.JudgedRecreates: "recreates a container",
delivery.JudgedMovesData: "moves the module's data",
delivery.JudgedCannot: "cannot be judged: " + need.Words,
}[need.Judgement]
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, 0),
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
Options: []asks.Option{
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the node keeps what it runs",
About: key,
}}
}
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
func NextBackupAsking(after int) int {
if after < 0 {
return 0
}
return max(after+1, 2)
}
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
// under a new number (ADR 0539 §16).
func BackupAskDigest(a delivery.AssignerAsk) string {
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
return m[1]
}
return ""
}
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
func BackupAnswerWords(option string, asking int) string {
return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
}
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
// backup's own words, from the node's have.
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
module, short(fp), node, words, module, digest),
Options: []asks.Option{
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(7 * 24 * time.Hour),
OnExpiry: "the module keeps running what it has",
About: key,
}}
}
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
// naming the node, the module and the backup's words; cleared under the same key.
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
}
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
needs := "answer whether to back up again or go on without a backup"
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
}
modules/mesh-delivery-assigner/internal/assign/asks_test.go
-98
View File
@@ -1,11 +1,9 @@
package assign
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -28,99 +26,3 @@ func TestTheHoldAskIsValid(t *testing.T) {
t.Errorf("the view may not answer it: %v", err)
}
}
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
func TestTheRestoreAskIsValid(t *testing.T) {
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
now := time.Now()
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
if err := a.Ask.Check(now); err != nil {
t.Errorf("the router refuses the ask: %v", err)
}
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("the ask cannot be written: %v", err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("the view may not answer it: %v", err)
}
var ids []string
for _, o := range a.Ask.Options {
ids = append(ids, o.ID)
}
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
t.Errorf("options = %v", ids)
}
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
}
}
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
// name (ADR 0437 clause 9), and is cleared under the same key.
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
}
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
t.Errorf("condition = %+v", c)
}
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
t.Errorf("cleared = %+v", got)
}
}
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
now := time.Now()
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
if err != nil {
t.Fatal(err)
}
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
if err != nil {
t.Fatal(err)
}
dgst := "sha256:" + strings.Repeat("b", 64)
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
if err := a.Ask.Check(now); err != nil {
t.Errorf("%s: the router refuses the ask: %v", key, err)
}
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
t.Errorf("%s: cannot be written: %v", key, err)
}
if _, err := delivery.ViewAnswerKey(key); err != nil {
t.Errorf("%s: the view may not answer it: %v", key, err)
}
}
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
t.Errorf("the record ask = %+v", r.Ask)
}
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
t.Errorf("the backup ask = %+v", b.Ask)
}
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
t.Errorf("asked again under the same id")
}
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
}
if BackupAskDigest(b) != dgst {
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
}
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
t.Errorf("two answers of the same option are one want value")
}
c := BackupCondition("ace", "mail", fNew, "no space left", true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Errorf("the backup condition is refused: %v", err)
}
}
modules/mesh-delivery-assigner/internal/assign/compare.go
+52
View File
@@ -0,0 +1,52 @@
package assign
import (
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its
// assignment names, and that build's fingerprint of the new form (empty when its record names none).
type Sent struct {
Sends bool
Build string
FP string
}
// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4),
// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and
// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the
// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said
// in words, for acceptance test 17 of design 47 to read.
func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) {
var d delivery.Difference
removed := want != nil && want.Removed
if want != nil && !removed {
d.Want = want.FP
}
d.WantRemoved = removed
if !sent.Sends {
if want == nil || removed {
return delivery.Difference{}, false
}
d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing"
return d, true
}
d.Sent, d.SentBuild = sent.FP, sent.Build
switch {
case sent.FP == "":
d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build)
case removed:
d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP))
case sent.FP == d.Want:
return delivery.Difference{}, false
case hold.Rule != 0 && sent.FP == newest:
d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words)
case d.Want == "":
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP))
default:
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want))
}
return d, true
}
modules/mesh-delivery-assigner/internal/assign/compare_test.go
+59
View File
@@ -0,0 +1,59 @@
package assign
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3.
func TestADifferenceExplainedByTheCanary(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out := Wants(m, n, wantedOld(), nil)
want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary}
d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"])
if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew {
t.Fatalf("difference = %+v, %v; want rule 3", d, ok)
}
if _, err := delivery.EncodeDifference(d); err != nil {
t.Errorf("the difference cannot be written: %v", err)
}
}
// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written
// unexplained, with words, and is writable.
func TestCompare(t *testing.T) {
w := delivery.Want{FP: fNew}
if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok {
t.Errorf("an equal want and send is a difference")
}
if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok {
t.Errorf("no want and nothing sent is a difference")
}
removed := delivery.Want{Removed: true}
if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok {
t.Errorf("a removed want and nothing sent is a difference")
}
old := delivery.Want{FP: fOld}
for name, c := range map[string]struct {
want *delivery.Want
sent Sent
}{
"sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}},
"names no fingerprint": {&w, Sent{Sends: true, Build: "b"}},
"sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}},
"sends nothing": {&w, Sent{}},
"no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}},
"no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}},
} {
d, ok := Compare(c.want, c.sent, fNew, Holding{})
if !ok || !d.Unexplained || d.Words == "" {
t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok)
continue
}
if _, err := delivery.EncodeDifference(d); err != nil {
t.Errorf("%s: the difference cannot be written: %v", name, err)
}
}
}
modules/mesh-delivery-assigner/internal/assign/wants.go
+9 -135
View File
@@ -20,24 +20,6 @@ type Module struct {
Granted bool
Manifest catalogue.Manifest
Nodes []string
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
Released bool
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
Restored func(node string) bool
}
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
if m.Composed == nil {
return delivery.ComposedDigest{}, false
}
return m.Composed(node, fp)
}
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
@@ -90,9 +72,6 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) {
if !m.Granted {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
}
if _, ok := m.composition(node, m.FP); !ok {
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
}
return true, 0, ""
}
@@ -138,24 +117,10 @@ type Outcome struct {
Wants map[string]delivery.Want
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
Held *delivery.Held
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
// back: the operator is asked whether to restore each (ADR 0539 §19).
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
// Rules is, by node, the rule holding the node back from m.FP and why, for the comparison.
Rules map[string]Holding
}
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
// controller's judgement of the move and its words.
type RecordNeed struct {
Node string
Digest string
Judgement delivery.Judgement
Words string
}
// Holding is a rule holding a node back, and why.
type Holding struct {
Rule int
@@ -165,12 +130,12 @@ type Holding struct {
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
// module holding data is not set back but asked about (Restore). A
// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
held := m.Held
if !held && !m.Released && m.FP != "" {
if !held && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
out.Held = &delivery.Held{Node: node, Words: h.Detail}
@@ -187,57 +152,30 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
before, had := wants[node]
var next delivery.Want
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
if w, ok := m.userListRevision(node, before, had, n); ok {
if !had || !delivery.SameWant(normal(before), normal(w)) {
out.Wants[node] = w
}
continue
}
switch ok, rule, words := Holds(mm, node, n); {
case ok:
d, _ := m.composition(node, m.FP)
next = delivery.Want{FP: m.FP, Composed: d.Digest}
if had && !before.Removed && before.FP == m.FP && before.Composed == d.Digest {
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
}
running, ran := n.Haves[node][m.Name]
if need, wait := m.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
}
continue
}
next = delivery.Want{FP: m.FP}
case held && had && before.FP == m.FP:
out.Rules[node] = Holding{Rule: rule, Words: words}
last := lastHealthy[node]
switch {
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
case keepsData(m.Manifest):
next = before
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
out.Restore = append(out.Restore, node)
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP))
case last == "":
next = before
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
default:
d, there := m.composition(node, last)
if !there {
next = before
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
break
}
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words}
}
default:
out.Rules[node] = Holding{Rule: rule, Words: words}
if !had || before.Removed {
continue
}
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words}
if before.FP == m.FP {
next = delivery.Want{FP: m.FP, Composed: before.Composed}
next = delivery.Want{FP: m.FP}
}
}
if !had || !delivery.SameWant(normal(before), normal(next)) {
@@ -265,67 +203,3 @@ func short(fp string) string {
}
return part[:12]
}
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
// same fingerprint judged neither (a user list only, §7).
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
return nil, false
}
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
prev := before.FP
if !had {
prev = running
}
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if m.Allowed != nil && m.Allowed(node, d.Digest) {
return nil, false
}
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
if d.Policy != "" {
return d.Policy == delivery.PolicyRecord
}
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
return policy == catalogue.PolicyRecord
}
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
if !catalogue.ProvidesBus(m.Manifest) {
return delivery.Want{}, false
}
running, ran := n.Haves[node][m.Name]
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
return delivery.Want{}, false
}
d, ok := m.composition(node, running.FP)
if !ok || !d.UserListOnly {
return delivery.Want{}, false
}
w := delivery.Want{FP: running.FP, Composed: d.Digest}
if had && before.Composed == d.Digest {
w.Backup, w.Words = before.Backup, before.Words
}
return w, true
}
modules/mesh-delivery-assigner/internal/assign/wants_test.go
+6 -389
View File
@@ -1,7 +1,6 @@
package assign
import (
"slices"
"strings"
"testing"
@@ -19,7 +18,7 @@ var (
// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider
// store healthy on ace and the canary ace running fNew healthy.
func ready() (Module, Nodes) {
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"},
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Nodes: []string{"g14", "ace"},
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
n := Nodes{
Online: map[string]bool{"ace": true, "g14": true},
@@ -32,16 +31,8 @@ func ready() (Module, Nodes) {
return m, n
}
// dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated.
func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) }
// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither.
func composedEverywhere(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, true
}
func wantedOld() map[string]delivery.Want {
return map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}, "g14": {FP: fOld, Composed: dig(fOld)}}
return map[string]delivery.Want{"ace": {FP: fOld}, "g14": {FP: fOld}}
}
// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so
@@ -50,7 +41,7 @@ func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
m, n := ready()
out := Wants(m, n, wantedOld(), nil)
for _, node := range []string{"ace", "g14"} {
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 {
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
}
}
@@ -68,9 +59,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
},
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
delivery.RuleComposed: func(m *Module, _ *Nodes) {
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
},
} {
m, n := ready()
breakIt(&m, &n)
@@ -110,11 +98,11 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}, "g14": {FP: fOld}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
t.Fatalf("held = %+v", out.Held)
}
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
if w := out.Wants["ace"]; w.FP != fOld || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w)
}
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
@@ -134,20 +122,6 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") {
t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w)
}
// 1906/D3: and the node is to be asked whether to restore it
if !slices.Equal(out.Restore, []string{"ace"}) {
t.Errorf("restore asks = %v, want ace", out.Restore)
}
}
// 1906/D3: a module without data is set back and asked about nothing.
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
if len(out.Restore) != 0 {
t.Errorf("restore asks = %v", out.Restore)
}
}
// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that
@@ -155,7 +129,7 @@ func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
m, n := ready()
m.Nodes = []string{"ace"}
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, nil)
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew}, "g14": {FP: fOld}}, nil)
if w := out.Wants["g14"]; !w.Removed {
t.Errorf("g14's want = %+v, want removed", w)
}
@@ -163,360 +137,3 @@ func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"])
}
}
// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its
// digest once it exists; a node with no want before still gets none until then.
func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
m, n := ready()
have := map[string]bool{}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if !have[node+"|"+fp] {
return delivery.ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, wantedOld(), nil)
for node, w := range out.Wants {
if w.FP == fNew {
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
}
}
if r := out.Rules["ace"]; r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "ace") {
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
}
// and a node that never had a want gets none
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
t.Errorf("wants without a composition: %+v", out.Wants)
}
have["ace|"+fNew] = true
out = Wants(m, n, wantedOld(), nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
}
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
t.Errorf("g14, whose composition does not exist, moved: %+v", w)
}
}
// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want,
// by the same rules: written when they hold, and not while one does not.
func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
m, n := ready()
revised := "sha256:" + strings.Repeat("9", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedNeither}, true
}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
out := Wants(m, n, running, nil)
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.FP != fNew || w.Composed != revised {
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
}
}
m.Granted = false
out = Wants(m, n, running, nil)
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
t.Errorf("a revision was written while a rule holds: %+v", w)
}
}
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if fp == fOld {
return delivery.ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.FP == fOld {
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
}
m.Composed = composedEverywhere
out = Wants(m, n, running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
t.Errorf("set-back want = %+v", w)
}
}
// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F).
func movingTo(j delivery.Judgement) (Module, Nodes, map[string]delivery.Want) {
m, n := ready()
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: j}
if j != delivery.JudgedNeither {
d.Against, d.Words = dig(fOld), "judged"
}
return d, true
}
return m, n, wantedOld()
}
func recording(m *Module) {
m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"}
}
// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be
// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint.
func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) {
for _, j := range []delivery.Judgement{delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot} {
m, n, wanted := movingTo(j)
recording(&m)
out := Wants(m, n, wanted, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
}
var ace *RecordNeed
for i := range out.Record {
if out.Record[i].Node == "ace" {
ace = &out.Record[i]
}
}
if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j {
t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record)
}
// a new revision at the same fingerprint is asked too
same := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}}
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
}
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
out = Wants(m, n, wanted, nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: after the yes ace's want = %+v", j, w)
}
}
}
// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module
// that rolls (not record) is never asked.
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
m, n, wanted := movingTo(delivery.JudgedNeither)
recording(&m)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("neither: %+v", out)
}
m, n, _ = movingTo(delivery.JudgedRecreates)
recording(&m)
delete(n.Haves["ace"], "mail")
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("a node that never ran it: %+v", out)
}
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
m, n, _ = movingTo(delivery.JudgedCannot)
recording(&m)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew || len(out.Record) == 0 {
t.Errorf("a node that runs it with no want: %+v", out)
}
m, n, wanted = movingTo(delivery.JudgedRecreates)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("a rolled module: %+v", out)
}
}
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
out := Wants(m, n, running, nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
}
if len(out.Record) != 0 {
t.Errorf("a user-list revision was asked: %+v", out.Record)
}
}
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
m, n := ready()
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew), Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
t.Errorf("the answer was erased: %+v", w)
}
}
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
// wants move by the rules again.
func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
m.Released = true
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleNotHeld}}, map[string]string{"ace": fOld})
if out.Held != nil {
t.Errorf("a released fingerprint was held again: %+v", out.Held)
}
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) || w.Rule != 0 {
t.Errorf("ace's want = %+v, want the held fingerprint written again", w)
}
}
// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes;
// answered "leave stopped" (or not answered) it stays.
func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
m.Restored = func(node string) bool { return false }
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.FP == fOld {
t.Errorf("set back without the operator's restore: %+v", w)
}
m.Restored = func(node string) bool { return node == "ace" }
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
t.Errorf("after restore ace's want = %+v", w)
}
}
// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included;
// a composition with none (made by hand) falls back to what the manifest derives.
func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
m, n, wanted := movingTo(delivery.JudgedMovesData)
with := func(policy, from string) {
inner := m.Composed
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d, ok := inner(node, fp)
d.Policy, d.PolicyFrom = policy, from
return d, ok
}
}
with(delivery.PolicyRecord, "person") // a roll module the operator set to record
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].FP == fNew {
t.Errorf("the operator's record policy was not honoured: %+v", out)
}
recording(&m) // the manifest says record, the operator chose roll
with("roll", "person")
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
t.Errorf("the operator's roll policy was not honoured: %+v", out)
}
with("", "") // made by hand: the manifest decides
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
}
}
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
}
}
// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written
// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it
// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against
// the composition the node runs, policy record, UserListOnly set.
func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedRecreates)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
userList := "sha256:" + strings.Repeat("7", 64)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
Policy: delivery.PolicyRecord, PolicyFrom: "the bus"}
if fp == fOld {
d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list
}
return d, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
for name, wanted := range map[string]map[string]delivery.Want{
"a want at the fingerprint it runs": {"ace": {FP: fOld, Composed: dig(fOld)}},
"no readable want": nil,
} {
out := Wants(m, n, wanted, nil)
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != userList {
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
}
if len(out.Record) != 0 {
t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record)
}
}
// the same flag on a module that is not the bus changes nothing
m.Manifest.Provides = nil
if out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
}
}
// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine
// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F
// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the
// declaration it kept); a new fingerprint still waits for its ask.
func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
for name, bus := range map[string]bool{"a record module": false, "the bus": true} {
// only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold
unchanged := func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedUnchanged}, true
}
m, n, _ := movingTo(delivery.JudgedCannot)
recording(&m)
m.Nodes = []string{"ace"}
if bus {
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
m.Composed = unchanged
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} // no composition named
out := Wants(m, n, nil, nil)
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
}
if len(out.Record) != 0 {
t.Errorf("%s: asked at the switch: %+v", name, out.Record)
}
// a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge,
// still wait for the ask
for what, j := range map[string]delivery.Judgement{"differing": delivery.JudgedRecreates, "unjudged": delivery.JudgedCannot, "judged neither": delivery.JudgedNeither} {
m2, n2, _ := movingTo(j)
recording(&m2)
m2.Nodes = []string{"ace"}
if bus {
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
n2.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out := Wants(m2, n2, nil, nil)
if j == delivery.JudgedNeither {
if out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
t.Errorf("%s, %s: %+v", name, what, out)
}
continue
}
if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 {
t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record)
}
}
// a have that names a composition is judged as before
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew && len(out.Record) == 0 {
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
}
// a new fingerprint still asks (the bus is barred instead)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, nil, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
}
if !bus && len(out.Record) == 0 {
t.Errorf("%s: a new fingerprint was not asked", name)
}
}
}
modules/mesh-delivery-assigner/manifest_test.go
+6 -7
View File
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
// its three states, and holds a credential of its own.
func TestManifest(t *testing.T) {
m := load(t)
if m.Module != "mesh-delivery-assigner" {
@@ -32,9 +32,9 @@ func TestManifest(t *testing.T) {
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants",
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
"mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
@@ -51,10 +51,9 @@ func TestManifest(t *testing.T) {
if m.SecretsOwner != "mesh-delivery-assigner" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
// checker puts its checks.
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
// 1622: the assigner puts its compose asks on the Go build seat, as the checker puts its checks.
if !slices.Equal(m.Uses, []string{"node-build-go"}) {
t.Errorf("uses = %v, want exactly node-build-go", m.Uses)
}
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
modules/mesh-delivery-assigner/module.json
+1 -4
View File
@@ -6,19 +6,16 @@
"mesh-build-go.steps",
"mesh-controller.assignments",
"mesh-controller.builds",
"mesh-controller.composed",
"mesh-controller.grants",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",
"node-engine.have"
],
"uses": [
"node-build-go",
"operator-channel"
"node-build-go"
],
"state": [
{ "name": "asks", "history": 5 },
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go
+1 -8
View File
@@ -121,12 +121,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
if err != nil {
return catalogue.Manifest{}, err
}
return parseManifest(kept.JSON)
}
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
func parseManifest(raw string) (catalogue.Manifest, error) {
return catalogue.ParseManifestForInputs([]byte(raw))
return catalogue.ParseManifest([]byte(kept.JSON))
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go
+2 -1
View File
@@ -6,6 +6,7 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
@@ -92,7 +93,7 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
if mk, err := delivery.ManifestsKey(m.Manifest); err == nil {
if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found {
if kept, err := delivery.DecodeManifest(raw); err == nil {
if man, err := parseManifest(kept.JSON); err == nil && man.Build != nil {
if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil {
for _, a := range man.Build.Artifacts {
if a.Language != "" {
set[a.Language] = true
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go
-29
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
@@ -113,31 +112,3 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t
t.Errorf("another repository: %v", got)
}
}
// A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the
// catalogue's predefined.json this process never holds: the indexer and the assigner read it with
// catalogue.ParseManifestForInputs (issues 1751, 1906). xterm's module.json from mesh-catalog at 835bf22b.
func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
if err != nil {
t.Fatal(err)
}
manifest := string(raw)
data, err := delivery.EncodeManifest(delivery.Manifest{Module: "xterm", JSON: manifest})
if err != nil {
t.Fatal(err)
}
store := memManifests{delivery.ManifestDigest([]byte(manifest)): data}
var said []string
r := &repoModules{manifests: store, logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
putModule(t, r, "xterm", "novox/mesh-catalog", manifest)
r.BuildModules("novox/mesh-catalog")
for _, s := range said {
if strings.Contains(s, "cannot be read") {
t.Errorf("the manifest was not read: %s", s)
}
}
if _, err := parseManifest(manifest); err != nil {
t.Errorf("manifestReader: %v", err)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/testdata/catalogue-835bf22b-xterm.module.json
Vendored
-236
View File
@@ -1,236 +0,0 @@
{
"module": "xterm",
"version": "1",
"moved": true,
"capabilities": [
"package-manager"
],
"requires": [
"x11-display"
],
"claims": [
{
"name": "node-terminal-emulator",
"scope": "node",
"serves": [
"open"
]
}
],
"tools": [
"xterm_font",
"xterm_colours"
],
"config": {
"face": {
"type": "string",
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
"pattern": "[A-Za-z0-9 ._-]{1,64}",
"default": "JetBrainsMono Nerd Font"
},
"size": {
"type": "number",
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
"minimum": 6,
"maximum": 72,
"predefined": "font-size",
"default": 12
},
"save-lines": {
"type": "number",
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
"minimum": 0,
"maximum": 1000000,
"default": 10000
},
"term-name": {
"type": "string",
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
"pattern": "[A-Za-z0-9._-]{1,64}",
"default": "xterm-256color"
},
"render-font": {
"type": "boolean",
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
"default": true
},
"bold-mode": {
"type": "boolean",
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
"default": false
},
"always-bold-mode": {
"type": "boolean",
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
"default": false
},
"allow-bold-fonts": {
"type": "boolean",
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
"default": true
},
"background": {
"type": "string",
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"foreground": {
"type": "string",
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a8a8a8"
},
"color0": {
"type": "string",
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#000000"
},
"color1": {
"type": "string",
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#9e1828"
},
"color2": {
"type": "string",
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#008800"
},
"color3": {
"type": "string",
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#d2bb4b"
},
"color4": {
"type": "string",
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#414171"
},
"color5": {
"type": "string",
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#963c59"
},
"color6": {
"type": "string",
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#418179"
},
"color7": {
"type": "string",
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bebebe"
},
"color8": {
"type": "string",
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#666666"
},
"color9": {
"type": "string",
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#bc5766"
},
"color10": {
"type": "string",
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#61a171"
},
"color11": {
"type": "string",
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#e7db52"
},
"color12": {
"type": "string",
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#5085af"
},
"color13": {
"type": "string",
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#a97a99"
},
"color14": {
"type": "string",
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#6ba4a4"
},
"color15": {
"type": "string",
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
"pattern": "#[0-9a-fA-F]{6}",
"default": "#ffffff"
},
"copy-binding": {
"type": "string",
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> C"
},
"paste-binding": {
"type": "string",
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
"pattern": "[^:\\\\\\n]+",
"default": "Ctrl Shift ~Alt <Key> V"
}
},
"items": [
{
"collection": "environment",
"name": "terminal",
"attributes": {
"kind": "variable",
"name": "TERMINAL"
},
"template": "files/env-terminal"
},
{
"collection": "shell:xresources:normal",
"name": "xresources-normal",
"template": "files/xresources-normal"
}
],
"resources": [
{
"id": "package",
"type": "package",
"package": "xterm"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/xterm-tools",
"binary": "xterm-tools",
"loads": [
"xterm-tools"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-checker/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-checker/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d h1:oJREw0ggKqJWJ5DEISSL0i2ZC5XqdXAidxo9NJ74TCc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-checker/internal/check/every_failure_test.go
+90
View File
@@ -0,0 +1,90 @@
package check
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// failingResult is a level's test result in which the named tests failed, each in its own package.
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
t.Helper()
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
if err != nil {
t.Fatal(err)
}
return raw
}
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
t.Helper()
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
return v, st
}
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
for name, tests := range map[string][]json.RawMessage{
"one result": {failingResult(t, 10, a, b)},
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
} {
v, st := judgeLevel(t, tests...)
if v == nil {
t.Fatalf("%s: the level passed", name)
}
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
}
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
t.Errorf("%s: verdict %+v", name, v)
}
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
t.Errorf("%s: %d tests counted, want all 10", name, run)
}
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
t.Errorf("%s: the verdict is not written: %v", name, err)
}
}
}
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
// failed, and each entry's line is cut to MaxFailureLine.
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
var fs []Failure
for i := range 60 {
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
}
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
}
if len(v.Failures[0].Words) > MaxFailureLine {
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
}
if n := failedCount(*v); n != 60 {
t.Errorf("failedCount = %d, want 60", n)
}
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
t.Errorf("line %q", line)
}
}
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
t.Fatalf("%q", got)
}
}
modules/mesh-delivery-checker/internal/check/examiner_test.go
+5 -3
View File
@@ -341,7 +341,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
}
}
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
// after it (so one check names every failure), and no later level runs.
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
@@ -357,8 +359,8 @@ func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { //
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
}
if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
modules/mesh-delivery-checker/internal/check/judge.go
+59 -17
View File
@@ -438,8 +438,9 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
}
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
// verdict. It answers the level's running time (its tests' and setup's).
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
// running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
level := la.Level
a := st.levels[level]
@@ -461,6 +462,7 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
a.kept += k.Tests
}
elapsed := 0.0
var failedResults []Tested
for _, raw := range la.Tests {
var res Tested
if err := json.Unmarshal(raw, &res); err != nil {
@@ -489,9 +491,12 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
}
}
if !res.Passed {
return levelFailure(t, level, res), elapsed
failedResults = append(failedResults, res)
}
}
if len(failedResults) > 0 {
return levelFailure(t, level, failedResults...), elapsed
}
return nil, elapsed
}
@@ -558,23 +563,60 @@ func firstLine(s string) string {
return ""
}
// failure turns a level that did not pass into a verdict naming what failed first.
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
const MaxFailureLine = 300
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
// saying how many more (issue 1921).
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
v := delivery.Verdict{State: delivery.VerdictFail}
switch {
case len(res.Failed) > 0:
f := res.Failed[0]
name := failedName(f)
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
var all []delivery.FailedFirst
for _, res := range results {
if len(res.Failed) == 0 {
why := res.Why
if why == "" {
why = "the " + level + " level did not pass"
}
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
continue
}
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
default:
why := res.Why
if why == "" {
why = "the " + level + " level did not pass"
for _, f := range res.Failed {
name := failedName(f)
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
}
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = failureLines(all)
return &v
}
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
n := min(len(all), delivery.MaxFailures)
out := make([]delivery.FailedFirst, 0, n+1)
for _, f := range all[:n] {
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
}
if more := len(all) - n; more > 0 {
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
}
return out
}
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
func clipLine(s string, n int) string {
if len(s) <= n {
return s
}
s = strings.ToValidUTF8(s[:n-3], "")
return s + "..."
}
modules/mesh-delivery-checker/internal/check/line.go
+31
View File
@@ -49,6 +49,34 @@ func firstFailed(v delivery.Verdict) string {
return firstWordsLine(f.Words)
}
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
func failedLines(v delivery.Verdict) []string {
if len(v.Failures) == 0 {
if f := firstFailed(v); f != "" {
return []string{f}
}
return nil
}
lines := make([]string, 0, len(v.Failures))
for _, f := range v.Failures {
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
}
return lines
}
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
func failedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
var more int
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
return delivery.MaxFailures + more
}
}
return n
}
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
func firstWordsLine(w string) string { return firstLine(w) }
@@ -155,6 +183,9 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
pc.head = "pass " + commits
default:
pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
}
pc.commits = commits
}
var counts []string
modules/mesh-delivery-checker/internal/check/try.go
+4 -4
View File
@@ -67,7 +67,9 @@ type Try struct {
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
Said string `json:"said,omitempty"`
// Took is the time from its start to its end, as Go writes a duration.
Took string `json:"took,omitempty"`
Took string `json:"took,omitempty"`
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
// (issue 1921); or what else failed first (a conflict, a refused step).
Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"`
@@ -339,9 +341,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
t.State = TryPass
default:
t.State = TryFail
if f := firstFailed(v); f != "" {
t.Failed = []string{f}
}
t.Failed = failedLines(v)
}
if t.Tests != "" {
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go
+1 -44
View File
@@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"os"
"strings"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-view/internal/view"
@@ -83,50 +82,9 @@ func Tools(r view.Reader) []stdio.Tool {
commit, _ := args["commit"].(string)
return r.BuilderBuild(module, fp2, commit)
}},
{Name: view.WantsReadTool,
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
return r.Wants(node, module)
}},
{Name: view.WantReadTool,
Description: "One want of the assigner as stored: what the node is to run of the module (fingerprint, the newest it is held back from and the rule, removed, words). Read-only.",
Input: map[string]any{"type": "object", "required": []string{"node", "module"}, "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
return r.Want(node, module)
}},
{Name: view.DifferencesReadTool,
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module. Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
return r.Differences(node, module)
}},
{Name: view.HavesReadTool,
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
"and <node>.node (the node's own facts). Narrow by node and module. Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
return r.Haves(node, module)
}},
}
}
var nodeModule = map[string]any{
"node": map[string]any{"type": "string", "description": "only this node's keys"},
"module": map[string]any{"type": "string", "description": "only this module's keys"}}
func nodeModuleOf(args map[string]any) (string, string) {
node, _ := args["node"].(string)
module, _ := args["module"].(string)
return strings.TrimSpace(node), strings.TrimSpace(module)
}
func read(state stdio.KeptState) view.StateRead {
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
e, err := state.Get(key)
@@ -144,8 +102,7 @@ func read(state stdio.KeptState) view.StateRead {
}
func main() {
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))}
if err := stdio.Serve("", Tools(r)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
os.Exit(1)
modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go
-65
View File
@@ -264,68 +264,3 @@ func TestCandidateToolsAnswerTheStoredValue(t *testing.T) {
t.Errorf("by module = %v", c)
}
}
// 1906/D1 AC1: assigner_wants, assigner_want, assigner_differences and node_haves answer as stored, narrowed by node and module.
func keptReader() view.Reader {
st := func(f fake) view.StateRead { return f.reader().State }
return view.Reader{
WantsState: st(fake{
"want.ace.zsh": `{"fp":"` + fp2A + `"}`,
"want.novox.zsh": `{"fp":"` + fp2B + `"}`,
"want.ace.netcheck": `{"fp":"` + fp2A + `","rule":3}`,
"held." + keyB: `{"node":"ace","words":"failed"}`,
}),
DifferencesState: st(fake{"ace.zsh": `{"unexplained":true}`, "novox.zsh": `{"rule":3}`}),
HavesState: st(fake{"ace.zsh": `{"fp":"x"}`, "ace.node": `{"facts":1}`, "novox.zsh": `{"fp":"y"}`}),
}
}
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
r := keptReader()
out, err := run(t, r, view.WantsReadTool, map[string]any{})
if err != nil {
t.Fatal(err)
}
a := out.(view.WantsAnswer)
if len(a.Wants) != 3 || len(a.Held) != 1 || string(a.Wants["want.ace.zsh"]) != `{"fp":"`+fp2A+`"}` {
t.Errorf("all = %+v", a)
}
out, _ = run(t, r, view.WantsReadTool, map[string]any{"node": "ace"})
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
t.Errorf("node ace = %v", got)
}
out, _ = run(t, r, view.WantsReadTool, map[string]any{"module": "zsh"})
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
t.Errorf("module zsh = %v", got)
}
}
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
r := keptReader()
out, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "zsh"})
if err != nil || string(out.(json.RawMessage)) != `{"fp":"`+fp2A+`"}` {
t.Errorf("want = %v %v", out, err)
}
if _, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "gone"}); err == nil || !strings.Contains(err.Error(), "no want") {
t.Errorf("a missing want: %v", err)
}
}
func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
r := keptReader()
out, err := run(t, r, view.DifferencesReadTool, map[string]any{"node": "novox"})
if err != nil {
t.Fatal(err)
}
if got := out.(view.KeptAnswer).Keys; len(got) != 1 || string(got["novox.zsh"]) != `{"rule":3}` {
t.Errorf("differences = %v", got)
}
out, _ = run(t, r, view.HavesReadTool, map[string]any{"module": "zsh"})
if got := out.(view.KeptAnswer).Keys; len(got) != 2 {
t.Errorf("haves of zsh = %v", got)
}
out, _ = run(t, r, view.HavesReadTool, map[string]any{"node": "ace"})
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
t.Errorf("haves of ace (the node's own key too) = %v", got)
}
}
modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go
+10 -237
View File
@@ -433,17 +433,17 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerWantsBucket, History: delivery.AssignerWantsHistory})
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerDifferencesBucket, History: delivery.AssignerDifferencesHistory})
if err != nil {
t.Fatal(err)
}
const fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
const fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
for key, d := range map[string]delivery.Want{
"want.ace.plex": {FP: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)},
"want.g14.plex": {FP: fpOld, Composed: "sha256:" + strings.Repeat("c", 64), Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"},
for key, d := range map[string]delivery.Difference{
"ace.plex": {Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"},
"g14.plex": {Want: fpOld, Sent: fpNew, Unexplained: true},
} {
v, err := delivery.EncodeWant(d)
v, err := delivery.EncodeDifference(d)
if err != nil {
t.Fatal(err)
}
@@ -465,13 +465,13 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
t.Fatalf("%s; last: %+v", what, last)
}
wait(func(s view.Snapshot) bool {
d := s.Wants
return len(d) == 2 && d[0].Node == "ace" && d[0].State == view.WantPending && d[1].Node == "g14" && d[1].State == view.WantHeldBack && d[1].Rule == delivery.RuleCanary
}, "the two wants never showed with where they stand")
if err := kv.Delete(ctx, "want.g14.plex"); err != nil {
d := s.Differences
return len(d) == 2 && d[0].Node == "ace" && d[0].Rule == delivery.RuleCanary && d[1].Node == "g14" && d[1].Unexplained
}, "the two differences never showed with their rules")
if err := kv.Delete(ctx, "g14.plex"); err != nil {
t.Fatal(err)
}
wait(func(s view.Snapshot) bool { return len(s.Wants) == 1 }, "the want taken away never left the list")
wait(func(s view.Snapshot) bool { return len(s.Differences) == 1 }, "the difference taken away never left the list")
}
// TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof starts the program whole, with the assigner's asks, the view's
@@ -696,230 +696,3 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC
}
t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog)
}
// 1906/D9, whole on the bus: when the wants and the haves make the source revision delivered, the view says it once, as its event
// `delivered` with the revision, the group and the heads, which the issue tracker derives "fix delivered" from.
func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) {
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}})
if err != nil {
t.Fatal(err)
}
kvs := map[string]jetstream.KeyValue{}
for _, name := range []string{delivery.SourceBucket, delivery.ModulesBucket, delivery.AssignerWantsBucket, delivery.HaveBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[name] = kv
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
fp := "src2:" + strings.Repeat("2", 64)
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
failed, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
good, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}})
for bucket, kv := range map[string][]byte{delivery.ModulesBucket: mod} {
if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil {
t.Fatal(err)
}
}
if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil {
t.Fatal(err)
}
if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", failed); err != nil {
t.Fatal(err)
}
rev, err := kvs[delivery.SourceBucket].Put(ctx, delivery.SourceMainKey, src)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second)
if info, err := events.Info(ctx); err != nil || info.State.Msgs != 0 {
t.Fatalf("an event was said before the revision was delivered: %v %+v", err, info.State)
}
if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", good); err != nil {
t.Fatal(err)
}
var msg *jetstream.RawStreamMsg
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && msg == nil; time.Sleep(100 * time.Millisecond) {
msg, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+DeliveredEventType)
}
if msg == nil {
t.Fatal("the delivery of the source revision was never said")
}
var e view.DeliveredEvent
if err := json.Unmarshal(msg.Data, &e); err != nil || e.Revision != rev || e.Group != "feat/x" || e.Heads["novox/web"] != strings.Repeat("1", 40) {
t.Errorf("the event = %+v, %v; want revision %d, group feat/x and the heads", e, err, rev)
}
if msg.Header.Get("x-source") != "mesh-delivery-view" || msg.Header.Get("x-event-id") == "" {
t.Errorf("the headers = %v", msg.Header)
}
}
// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and
// healthy `settled`, a have held or failed `stuck`, each once with the body the messenger reads and an event id per type, node, module
// and composition.
func TestProcessWholeSaysModulesInMotion(t *testing.T) {
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}})
if err != nil {
t.Fatal(err)
}
kvs := map[string]jetstream.KeyValue{}
for _, name := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
kvs[name] = kv
}
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
time.Sleep(2 * time.Second) // the view has read what stood: nothing
fpOld, fpNew := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64)
digest := "sha256:" + strings.Repeat("c", 64)
put := func(bucket, key string, v []byte, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
if _, err := kvs[bucket].Put(ctx, key, v); err != nil {
t.Fatal(err)
}
}
want := func(node, module string) {
v, err := delivery.EncodeWant(delivery.Want{FP: fpNew, Composed: digest})
put(delivery.AssignerWantsBucket, "want."+node+"."+module, v, err)
}
have := func(node, module, fp string, phase delivery.Phase, health delivery.Health, detail string) {
v, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: phase, Health: health, Detail: detail})
put(delivery.HaveBucket, node+"."+module, v, err)
}
have("ace", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "")
want("ace", "web")
have("g14", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "")
want("g14", "web")
said := func() map[string]jetstream.RawStreamMsg {
out := map[string]jetstream.RawStreamMsg{}
for seq := uint64(1); ; seq++ {
m, err := events.GetMsg(ctx, seq)
if err != nil {
return out
}
out[m.Header.Get("x-event-id")] = *m
}
}
waitFor := func(id string) jetstream.RawStreamMsg {
t.Helper()
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if m, ok := said()[id]; ok {
return m
}
}
t.Fatalf("never said %s; said %v", id, func() []string {
var ids []string
for k := range said() {
ids = append(ids, k)
}
return ids
}())
return jetstream.RawStreamMsg{}
}
hex := strings.Repeat("c", 64)
mv := waitFor("moving.ace.web." + hex)
var body view.MotionBody
if err := json.Unmarshal(mv.Data, &body); err != nil || body.Node != "ace" || body.Module != "web" || body.FromFP != fpOld || body.ToFP != fpNew || body.Composed != digest || body.BoundSeconds != view.MotionBoundSeconds || body.Since.IsZero() {
t.Errorf("moving = %+v, %v", body, err)
}
if mv.Subject != "mesh.mod.mesh-delivery-view.event.moving" {
t.Errorf("subject = %s", mv.Subject)
}
have("ace", "web", fpNew, delivery.PhaseRunning, delivery.HealthOK, "")
st := waitFor("settled.ace.web." + hex)
if err := json.Unmarshal(st.Data, &body); err != nil || body.Outcome != "running" {
t.Errorf("settled = %+v, %v", body, err)
}
have("g14", "web", fpOld, delivery.PhaseHeld, delivery.HealthWaits, "no space left")
stk := waitFor("stuck.g14.web." + hex)
if err := json.Unmarshal(stk.Data, &body); err != nil || body.Outcome != "stopped" || body.Words != "no space left" {
t.Errorf("stuck = %+v, %v", body, err)
}
time.Sleep(time.Second)
count := map[string]int{}
for id := range said() {
count[strings.SplitN(id, ".", 2)[0]]++
}
if count["moving"] != 2 || count["settled"] != 1 || count["stuck"] != 1 {
t.Errorf("events said by type = %v, want moving 2, settled 1, stuck 1", count)
}
}
modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go
+2 -38
View File
@@ -138,18 +138,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
model.SetWriter(kvWriter{ctx: ctx, js: js})
model.SetMotionEvent(func(e view.MotionEvent) {
if err := emitEvent(ctx, js, cred, e.Type, e.Body, e.ID); err != nil {
logger.Printf("the %s of %s on %s could not be said: %v", e.Type, e.Body.Module, e.Body.Node, err)
}
})
model.SetDeliveredEvent(func(e view.DeliveredEvent) {
if err := emitEvent(ctx, js, cred, DeliveredEventType, e, fmt.Sprintf("delivered.%d", e.Revision)); err != nil {
logger.Printf("the delivery of source revision %d could not be said: %v", e.Revision, err)
}
})
model.SetBacklog(view.BacklogOver(js, 5*time.Second))
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} {
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} {
go follow(ctx, js, bucket, model, logger)
}
select {
@@ -214,36 +204,10 @@ func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, model
return errors.New("the watch ended")
}
if e == nil { // the last values have all been delivered
model.Initial(bucket)
continue
}
model.Apply(view.Entry{Bucket: bucket, Key: e.Key(), Value: e.Value(), At: e.Created(),
Deleted: e.Operation() != jetstream.KeyValuePut, Revision: e.Revision()})
Deleted: e.Operation() != jetstream.KeyValuePut})
}
}
}
// DeliveredEventType is the event the view says when a source revision is delivered: mesh.mod.mesh-delivery-view.event.delivered
// (issue 1906 D9, ADR 0539 §17). Its body is view.DeliveredEvent.
const DeliveredEventType = "delivered"
// emitEvent publishes an event of the view's own namespace, mesh.mod.<module>.event.<type>, with the SDK's envelope headers
// (as mesh-sdk's process.Conn.Emit does) and the id as the bus's own dedup id, so the same revision said twice in the stream's
// duplicate window is kept once.
func emitEvent(ctx context.Context, js jetstream.JetStream, cred credential, eventType string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + eventType)
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id))
return err
}
modules/mesh-delivery-view/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-view/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-view/internal/view/differences_test.go
+103
View File
@@ -0,0 +1,103 @@
package view
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const (
fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
)
// diffEntry is a key of the assigner's differences as the assigner writes it (mesh-sdk go/delivery EncodeDifference).
func differenceE(t *testing.T, node, module string, d delivery.Difference) Entry {
t.Helper()
key, err := delivery.DifferenceKey(node, module)
if err != nil {
t.Fatal(err)
}
v, err := delivery.EncodeDifference(d)
if err != nil {
t.Fatal(err)
}
return Entry{Bucket: delivery.AssignerDifferencesBucket, Key: key, Value: v, At: t0}
}
func TestEveryDifferenceIsListedWithItsRule(t *testing.T) { // 1297/B8 AC
m := New()
fill(m,
differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}),
differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}),
differenceE(t, "ace", "old", delivery.Difference{WantRemoved: true, Sent: fpOld, Rule: delivery.RuleNotHeld}),
)
got := m.Snapshot().Differences
if len(got) != 3 {
t.Fatalf("differences = %+v, want all three", got)
}
// ordered by node then module
if got[0].Node != "ace" || got[0].Module != "old" || got[1].Module != "plex" || got[2].Node != "g14" {
t.Errorf("order = %+v", got)
}
if r := got[1]; r.Want != fpOld || r.Sent != fpNew || r.Rule != delivery.RuleCanary || r.Unexplained || r.Words != "ace is not the canary" {
t.Errorf("row = %+v", r)
}
if !got[0].WantRemoved || got[0].Rule != delivery.RuleNotHeld {
t.Errorf("removed row = %+v", got[0])
}
if !got[2].Unexplained || got[2].Rule != 0 {
t.Errorf("unexplained row = %+v", got[2])
}
// a difference taken away (the want and the send agree again) leaves the list
fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "g14.plex", Deleted: true})
if got := m.Snapshot().Differences; len(got) != 2 {
t.Errorf("after the delete = %+v", got)
}
}
func TestAnUnreadableDifferenceIsNamedAndNotDropped(t *testing.T) { // never dropped silently
m := New()
fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "ace.plex", Value: []byte(`{"v":99}`), At: t0})
s := m.Snapshot()
if len(s.Differences) != 0 || len(s.Unreadable) != 1 || s.Unreadable[0].Bucket != delivery.AssignerDifferencesBucket || s.Unreadable[0].Key != "ace.plex" {
t.Errorf("snapshot = %+v", s)
}
}
func TestTableAndPageShowTheWantsAgainstTheSends(t *testing.T) { // 1297/B8 AC
m := New()
fill(m,
differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}),
differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}),
)
var out strings.Builder
Table(&out, m.Snapshot())
for _, w := range []string{"WANTS AGAINST SENDS", "ace", "plex", fpOld, fpNew, "rule 3", "ace is not the canary", "unexplained"} {
if !strings.Contains(out.String(), w) {
t.Errorf("table lacks %q:\n%s", w, out.String())
}
}
for _, path := range []string{"/", "/wants"} {
rec := httptest.NewRecorder()
Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))
body := rec.Body.String()
if rec.Code != 200 {
t.Fatalf("%s = %d", path, rec.Code)
}
for _, w := range []string{"WANTS AGAINST SENDS", "rule 3", "unexplained", "ace is not the canary"} {
if !strings.Contains(body, w) {
t.Errorf("%s lacks %q:\n%s", path, w, body)
}
}
}
// With none, the page says so and does not hide the heading.
rec := httptest.NewRecorder()
Web(New()).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/wants", nil))
if !strings.Contains(rec.Body.String(), "no difference") {
t.Errorf("empty page = %q", rec.Body.String())
}
}
modules/mesh-delivery-view/internal/view/read.go
+1 -133
View File
@@ -33,10 +33,7 @@ type Reader struct {
State StateRead // the indexer's modules
Builds StateRead // the builder's builds
CandidatesState StateRead // the indexer's candidates
WantsState StateRead // the assigner's wants and holds
DifferencesState StateRead // the assigner's differences
HavesState StateRead // the node-engine's haves
CandidatesState StateRead // the indexer's candidates
}
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
@@ -331,132 +328,3 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
}
return row, nil
}
// The read tools of the assigner's wants and differences and of the node-engine's haves (issue 1906, D1): the keys of those
// buckets answered as stored, so what the assigner wants, what it found different and what each node has can be read live.
// They write nothing.
const (
WantsReadTool = "assigner_wants"
WantReadTool = "assigner_want"
DifferencesReadTool = "assigner_differences"
HavesReadTool = "node_haves"
)
// WantsAnswer is the answer of assigner_wants: the wants by key (`want.<node>.<module>`, a node's own `want.<node>.node`
// among them) and the holds by key
// (`held.<module>.<hex>`), each as stored, and the keys that are neither.
type WantsAnswer struct {
Wants map[string]json.RawMessage `json:"wants"`
Held map[string]json.RawMessage `json:"held"`
Unreadable []string `json:"unreadable,omitempty"`
}
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
// are no `<node>.<module>` key.
type KeptAnswer struct {
Keys map[string]json.RawMessage `json:"keys"`
Unreadable []string `json:"unreadable,omitempty"`
}
func (s StateRead) all(what string) (map[string]json.RawMessage, error) {
if s.Keys == nil || s.Get == nil {
return nil, fmt.Errorf("%s cannot be read here", what)
}
keys, err := s.Keys()
if err != nil {
return nil, err
}
sort.Strings(keys)
out := map[string]json.RawMessage{}
for _, k := range keys {
raw, err := s.Get(k)
if err != nil {
return nil, err
}
if len(raw) > 0 {
out[k] = raw
}
}
return out, nil
}
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
// matches).
func (r Reader) Wants(node, module string) (WantsAnswer, error) {
all, err := r.WantsState.all("the assigner's wants")
if err != nil {
return WantsAnswer{}, err
}
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
for k, raw := range all {
if n, m, err := delivery.ParseWantKey(k); err == nil {
if (node == "" || n == node) && (module == "" || m == module) {
out.Wants[k] = raw
}
continue
}
// A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node".
if n, err := delivery.ParseWantNodeKey(k); err == nil {
if (node == "" || n == node) && (module == "" || module == "node") {
out.Wants[k] = raw
}
continue
}
if m, _, err := delivery.ParseHeldKey(k); err == nil {
if node == "" && (module == "" || m == module) {
out.Held[k] = raw
}
continue
}
out.Unreadable = append(out.Unreadable, k)
}
return out, nil
}
// Want is the stored want of module on node.
func (r Reader) Want(node, module string) (json.RawMessage, error) {
key, err := delivery.WantKey(strings.TrimSpace(node), strings.TrimSpace(module))
if err != nil {
return nil, err
}
if r.WantsState.Get == nil {
return nil, fmt.Errorf("the assigner's wants cannot be read here")
}
raw, err := r.WantsState.Get(key)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, fmt.Errorf("the assigner has no want %s", key)
}
return raw, nil
}
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
all, err := s.all(what)
if err != nil {
return KeptAnswer{}, err
}
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
for k, raw := range all {
n, m, isNode, err := delivery.ParseHaveKey(k)
if err != nil {
out.Unreadable = append(out.Unreadable, k)
continue
}
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
out.Keys[k] = raw
}
}
return out, nil
}
// Differences is the assigner's differences as stored, narrowed by node and module.
func (r Reader) Differences(node, module string) (KeptAnswer, error) {
return r.kept(r.DifferencesState, "the assigner's differences", node, module)
}
// Haves is the node-engine's haves (and each node's own key) as stored, narrowed by node and module.
func (r Reader) Haves(node, module string) (KeptAnswer, error) {
return r.kept(r.HavesState, "the haves", node, module)
}
modules/mesh-delivery-view/internal/view/serve.go
+28 -43
View File
@@ -209,42 +209,33 @@ func askText(s Snapshot) [][]string {
return out
}
// wantText is one line per want: the node, the module, what the assigner wants, what the node runs, where the two stand and the
// assigner's words (the rule that holds a node back, when one does).
func wantText(s Snapshot) [][]string {
// differenceText is one line per difference: the node, the module, what the assigner wants, what the controller sends, the rule
// that explains it (or unexplained) and the assigner's words.
func differenceText(s Snapshot) [][]string {
var out [][]string
for _, w := range s.Wants {
want, have := w.Want, w.Have
if want == "" {
for _, d := range s.Differences {
want, sent := d.Want, d.Sent
switch {
case d.WantRemoved:
want = "removed"
case want == "":
want = "none"
}
if have == "" {
have = "none"
if sent == "" {
sent = "none"
if d.SentBuild != "" {
sent = "build " + d.SentBuild
}
}
words := w.Words
if w.Rule != 0 {
words = fmt.Sprintf("rule %d: %s", w.Rule, w.Words)
why := "unexplained"
if !d.Unexplained {
why = fmt.Sprintf("rule %d", d.Rule)
}
judged := w.Judgement
if judged == "" {
judged = "none"
} else if w.Judged != "" {
words = strings.TrimSpace(words + " " + w.Judged)
}
out = append(out, []string{w.Node, w.Module, want, have, w.State, judged, words})
out = append(out, []string{d.Node, d.Module, want, sent, why, d.Words})
}
return out
}
// sourceText is the source revision's derived state in one line, then one line for each module on a node it waits for.
func sourceText(s Snapshot) []string {
if s.Source == nil {
return nil
}
out := []string{fmt.Sprintf("source revision %d: %s", s.Source.Revision, s.Source.State)}
return append(out, s.Source.Waiting...)
}
// backlogText is one line per language and kind: its waiting asks, the asks held by a node, the oldest waiting ask's age.
func backlogText(s Snapshot) [][]string {
var out [][]string
@@ -263,7 +254,7 @@ func backlogText(s Snapshot) [][]string {
var backlogHead = []string{"LANGUAGE", "KIND", "WAITING", "HELD", "OLDEST WAITING"}
var wantHead = []string{"NODE", "MODULE", "WANT", "HAVE", "STATE", "MOVE JUDGED", "WORDS"}
var differenceHead = []string{"NODE", "MODULE", "WANT", "SENT", "RULE", "WORDS"}
var head = []string{"NODE", "MODULE", "ASSIGNED", "HAVE", "PHASE", "HEALTH", "SINCE", "DETAIL", "WAITS"}
@@ -312,16 +303,10 @@ func Table(w io.Writer, s Snapshot) {
row(cells)
}
}
if src := sourceText(s); len(src) > 0 {
if d := differenceText(s); len(d) > 0 {
fmt.Fprintln(tw)
for _, line := range src {
row([]string{line})
}
}
if d := wantText(s); len(d) > 0 {
fmt.Fprintln(tw)
row([]string{"WANTS AGAINST HAVES"})
row(wantHead)
row([]string{"WANTS AGAINST SENDS"})
row(differenceHead)
for _, cells := range d {
row(cells)
}
@@ -371,16 +356,16 @@ var page = template.Must(template.New("p").Parse(`<!doctype html><html><head><me
{{if .Groups}}<h2>CANDIDATES</h2><table><tr><th>GROUP</th><th>PULL REQUEST</th><th>HEAD</th><th>MODULES</th></tr>{{range .Groups}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Asks}}<h2>ASKS</h2><table><tr><th>ASK</th><th>HEADLINE</th><th>WORDS</th><th>PULL REQUESTS</th><th>ANSWER</th></tr>{{range .Asks}}<tr>{{range .Cells}}<td>{{.}}</td>{{end}}<td>{{if .Options}}<form method="post" action="/answer"><input type="hidden" name="key" value="{{.Key}}"><input name="words" placeholder="words (optional)">{{range .Options}} <button name="option" value="{{.}}">{{.}}</button>{{end}}</form>{{end}}</td></tr>{{end}}</table>{{end}}
{{if .Backlog}}<h2>BACKLOG OF BUILD ASKS</h2><table><tr>{{range .BacklogHead}}<th>{{.}}</th>{{end}}</tr>{{range .Backlog}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Source}}<h2>SOURCE REVISION</h2><p>{{range .Source}}{{.}}<br>{{end}}</p>{{end}}{{if .Wants}}<h2>WANTS AGAINST HAVES</h2><table><tr>{{range .WantHead}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Differences}}<h2>WANTS AGAINST SENDS</h2><table><tr>{{range .DifferenceHead}}<th>{{.}}</th>{{end}}</tr>{{range .Differences}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Fingerprints}}<h2>FINGERPRINTS</h2><table><tr><th>MODULE</th><th>STATE</th><th>INDEXER AGAINST BUILD</th></tr>{{range .Fingerprints}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Offline}}<h2>OFFLINE</h2><table>{{range .Offline}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Stuck}}<h2>STUCK</h2><table><tr><th>BUCKET</th><th>KEY</th><th>WAITED FOR</th><th>SINCE</th></tr>{{range .Stuck}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
{{if .Unreadable}}<h2>UNREADABLE</h2><table>{{range .Unreadable}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}</body></html>`))
var wantsPage = template.Must(template.New("w").Parse(`<!doctype html><html><head><meta charset="utf-8"><meta http-equiv="refresh" content="5">
<title>Wants against haves</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
<body><p><a href="/">What each node runs</a></p><h1>WANTS AGAINST HAVES</h1>
{{if .Wants}}<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
<title>Wants against sends</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
<body><p><a href="/">What each node runs</a></p><h1>WANTS AGAINST SENDS</h1>
{{if .Differences}}<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>{{range .Differences}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
{{else}}<p>no difference: every want is what the controller sends.</p>{{end}}</body></html>`))
// Web is the read-only web view of the same rows, refreshed by the browser.
@@ -402,7 +387,7 @@ func Web(m *Model) http.Handler {
}
if r.URL.Path == "/wants" {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
wantsPage.Execute(w, map[string]any{"Head": wantHead, "Wants": wantText(m.Snapshot())})
wantsPage.Execute(w, map[string]any{"Head": differenceHead, "Differences": differenceText(m.Snapshot())})
return
}
if r.URL.Path != "/" {
@@ -425,7 +410,7 @@ func Web(m *Model) http.Handler {
for _, u := range snap.Unreadable {
bad = append(bad, []string{u.Bucket, u.Key, u.Error})
}
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Wants": wantText(snap), "Source": sourceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "WantHead": wantHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Differences": differenceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "DifferenceHead": differenceHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
})
}
modules/mesh-delivery-view/internal/view/view.go
+61 -481
View File
@@ -21,8 +21,6 @@ type Entry struct {
Value []byte
At time.Time
Deleted bool
// Revision is the bucket revision of the change, when the watch gives one (the source revision is the gate's source key's).
Revision uint64
}
// Row is one module on one node.
@@ -68,59 +66,27 @@ type Snapshot struct {
Fingerprints []FingerprintRow `json:"fingerprints"`
// Asks is every ask the gate holds that has no answer yet: what waits for the operator's Approve.
Asks []AskRow `json:"asks"`
// Wants is every want of the assigner beside the have of its node and where it stands (novox/hq ADR 0539 §8, issue 1906 D7).
Wants []WantRow `json:"wants"`
// Source is the state of the mesh's source revision, derived from the wants and the haves (ADR 0429 §1): recorded by no part.
// Nil when the gate holds no source yet.
Source *SourceRow `json:"source,omitempty"`
// Differences is every difference between what the assigner wants a node to run and what the controller sends, with the rule that
// explains it or unexplained (novox/hq ADR 0528 clause 4, issue 1297 B8).
Differences []DifferenceRow `json:"differences"`
// Backlog is, for each language and kind of build ask with one waiting or held, the asks waiting for a node, the asks a node
// holds and the oldest waiting ask's age (issue 1887 A8a).
Backlog []BacklogRow `json:"backlog"`
}
// The states of a want against the have of its node.
const (
WantApplied = "applied" // the node runs the want's fingerprint, running and healthy
WantPending = "pending" // the node has not yet run the want's fingerprint, or not healthy yet
WantFailed = "failed" // the node runs the want's fingerprint and its health check failed
WantHeldBack = "held back" // the assigner holds the node back from the newest fingerprint (Rule), or set it back
WantRemoved = "removed" // the node's want is removed: the module is no longer assigned to it
)
// WantRow is a want of the assigner beside the have of its node: the fingerprint wanted, the one the node runs with its phase and
// health, the state of the two, and for a want held back the newest fingerprint, the rule and the assigner's words.
type WantRow struct {
Node string `json:"node"`
Module string `json:"module"`
Want string `json:"want,omitempty"`
Have string `json:"have,omitempty"`
Phase string `json:"phase,omitempty"`
Health string `json:"health,omitempty"`
State string `json:"state"`
Newest string `json:"newest,omitempty"`
Rule int `json:"rule,omitempty"`
Words string `json:"words,omitempty"`
// Judgement is the controller's judgement of the move to the composition the want names (recreates, moves-data, neither or
// cannot-judge), and Judged its words; empty when the controller wrote none (ADR 0539 §14).
Judgement string `json:"judgement,omitempty"`
Judged string `json:"judged,omitempty"`
}
// The states of a source revision (ADR 0429 §1).
const (
SourceDelivered = "delivered"
SourceNotDelivered = "not delivered"
)
// SourceRow is the mesh's source revision (the revision of the gate's source key) and whether it is delivered: every want equals its
// module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is assigned to
// has a want. Waiting names each module on a node that is not there yet, and why.
type SourceRow struct {
Revision uint64 `json:"revision"`
State string `json:"state"`
Waiting []string `json:"waiting,omitempty"`
// Wants is how many wants of modules at the indexer were looked at; a revision with none is delivered of nothing.
Wants int `json:"wants"`
// DifferenceRow is a module on a node whose want and send differ: the fingerprint wanted (empty when the node has no want,
// WantRemoved when the want is removed), the fingerprint of the build the controller's assignment names (empty when it sends
// none; SentBuild names that build), and the rule that explains it (Rule 1 to 6) or Unexplained.
type DifferenceRow struct {
Node string `json:"node"`
Module string `json:"module"`
Want string `json:"want,omitempty"`
WantRemoved bool `json:"want-removed,omitempty"`
Sent string `json:"sent,omitempty"`
SentBuild string `json:"sent-build,omitempty"`
Rule int `json:"rule,omitempty"`
Unexplained bool `json:"unexplained,omitempty"`
Words string `json:"words,omitempty"`
}
// AskRow is an open ask (novox/hq ADR 0437 §10): the gate's ask with no answer under its key in the teller's answers.
@@ -212,29 +178,22 @@ type UnreadableRow struct {
// Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values.
type Model struct {
mu sync.Mutex
assigns map[string]assignment
builds map[string]buildEntry // by <module>.build.<id>
groups map[string]groupEntry // the indexer's candidates, by group
modules map[string]moduleEntry // the indexer's modules, by module
keys map[string]keyEntry // the builder's build keys, by <module>.<fingerprint>
asks map[string]askRecord // the gate's asks, by key
wants map[string]wantEntry // the assigner's wants, by <node>.<module>
source *sourceEntry // the gate's source key
deliveredEvent func(DeliveredEvent)
motionEvent func(MotionEvent)
told map[string]bool // the motion events said, by their ids
initial map[string]bool // the buckets whose watch handed over what stood at its start
said uint64 // the source revision whose delivery was said
composed map[string]composedEntry // the controller's digests and judgements, by <node>.<module>.<hex>
aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
writer Writer
backlog *Backlog
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
newest map[string]string // module -> id
haves map[string]haveEntry
nodes map[string]nodeEntry
mu sync.Mutex
assigns map[string]assignment
builds map[string]buildEntry // by <module>.build.<id>
groups map[string]groupEntry // the indexer's candidates, by group
modules map[string]moduleEntry // the indexer's modules, by module
keys map[string]keyEntry // the builder's build keys, by <module>.<fingerprint>
asks map[string]askRecord // the gate's asks, by key
diffs map[string]diffEntry // the assigner's differences, by <node>.<module>
aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
writer Writer
backlog *Backlog
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
newest map[string]string // module -> id
haves map[string]haveEntry
nodes map[string]nodeEntry
// the watchman's buckets, by key
presence map[string]presenceEntry
stuck map[string]stuckEntry
@@ -297,22 +256,9 @@ type keyEntry struct {
err string
}
type wantEntry struct {
value delivery.Want
type diffEntry struct {
value delivery.Difference
err string
at time.Time
}
type composedEntry struct {
value delivery.ComposedDigest
err string
}
type sourceEntry struct {
revision uint64
group string
heads map[string]string
err string
}
type assignerAskRecord struct {
@@ -344,64 +290,18 @@ type nodeEntry struct {
// New is an empty model.
func New() *Model {
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{},
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, diffs: map[string]diffEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{},
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
}
// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed. When the change makes
// the source revision delivered, the event of it is said once (SetDeliveredEvent), after the model's lock is released.
// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed.
func (m *Model) Apply(e Entry) {
m.mu.Lock()
m.apply(e)
event := m.deliveredNow()
say := m.deliveredEvent
motion, sayMotion := m.motionNow(e), m.motionEvent
m.mu.Unlock()
if event != nil && say != nil {
say(*event)
}
if sayMotion != nil {
for _, ev := range motion {
sayMotion(ev)
}
}
}
// DeliveredEvent is what the view says when the source revision is delivered (event `delivered` of the view, issue 1906 D9, ADR
// 0539 §17): the revision of the gate's source key, the group that moved it and every repository's merged head at it. A
// consumer takes every earlier revision as covered by it, and takes the same revision twice as once (after a restart of the view).
type DeliveredEvent struct {
Revision uint64 `json:"revision"`
Group string `json:"group"`
Heads map[string]string `json:"heads"`
}
// SetDeliveredEvent sets where the event of a delivered source revision goes; nothing is said until it is set.
func (m *Model) SetDeliveredEvent(say func(DeliveredEvent)) {
m.mu.Lock()
defer m.mu.Unlock()
m.deliveredEvent = say
}
// deliveredNow is the event of the source revision when it is delivered now and has not been said, or nil. A revision that went
// bad after it was delivered and recovered is not said again.
func (m *Model) deliveredNow() *DeliveredEvent {
if m.deliveredEvent == nil || m.source == nil || m.source.err != "" || m.said == m.source.revision {
return nil
}
row := m.sourceRow()
if row == nil || row.State != SourceDelivered || row.Wants == 0 {
return nil
}
m.said = m.source.revision
return &DeliveredEvent{Revision: m.source.revision, Group: m.source.group, Heads: m.source.heads}
}
func (m *Model) apply(e Entry) {
switch e.Bucket {
case delivery.HaveBucket:
node, _, isNode, err := delivery.ParseHaveKey(e.Key)
node, module, isNode, err := delivery.ParseHaveKey(e.Key)
if err != nil {
return
}
@@ -414,7 +314,7 @@ func (m *Model) apply(e Entry) {
m.nodes[node] = nodeEntry{node: n, err: errText(err), at: e.At}
return
}
key := e.Key // the bucket's own key: a module name with a dot is escaped in it (issue 1150)
key := node + "." + module
if e.Deleted {
delete(m.haves, key)
return
@@ -485,38 +385,13 @@ func (m *Model) apply(e Entry) {
if a, err := delivery.DecodeViewAnswer(e.Value); err == nil {
m.chosen[askKey] = a
}
case delivery.AssignerWantsBucket:
node, module, err := delivery.ParseWantKey(e.Key)
if err != nil {
return // a hold, and the node's own key, are not a module's want
}
key, kerr := delivery.HaveKey(node, module)
if kerr != nil {
return
}
case delivery.AssignerDifferencesBucket:
if e.Deleted {
delete(m.wants, key)
delete(m.diffs, e.Key)
return
}
w, err := delivery.DecodeWant(e.Value)
m.wants[key] = wantEntry{value: w, err: errText(err), at: e.At}
case delivery.ControllerComposedBucket:
if e.Deleted {
delete(m.composed, e.Key)
return
}
d, err := delivery.DecodeComposedDigest(e.Value)
m.composed[e.Key] = composedEntry{value: d, err: errText(err)}
case delivery.SourceBucket:
if e.Key != delivery.SourceMainKey {
return
}
if e.Deleted {
m.source = nil
return
}
src, err := delivery.DecodeSource(e.Value)
m.source = &sourceEntry{revision: e.Revision, group: src.Group, heads: src.Heads, err: errText(err)}
d, err := delivery.DecodeDifference(e.Value)
m.diffs[e.Key] = diffEntry{value: d, err: errText(err)}
case delivery.AnswersBucket:
if e.Deleted {
delete(m.answers, e.Key)
@@ -612,17 +487,7 @@ func (m *Model) snapshot() Snapshot {
r.RequiredBy = a.RequiredBy
}
var waits []string
hk, herr := delivery.HaveKey(a.Node, a.Module)
if herr != nil {
hk = key
}
h, has := m.haves[hk]
// The controller no longer writes the build of an assignment (R6): the row is the want and the have of the node.
w, wanted := m.wants[hk]
wanted = wanted && w.err == "" && !w.value.Removed && w.value.FP != ""
if wanted {
r.Assigned = w.value.FP
}
h, has := m.haves[key]
switch {
case !has:
waits = append(waits, "no have yet")
@@ -630,15 +495,8 @@ func (m *Model) snapshot() Snapshot {
r.Since, r.Error = h.at, h.err
default:
r.Have, r.Phase, r.Health = h.have.Build, string(h.have.Phase), string(h.have.Health)
if r.Have == "" {
r.Have = h.have.FP
}
r.Since, r.Detail = h.at, h.have.Detail
if wanted {
if h.have.FP != w.value.FP {
waits = append(waits, "the node has not applied "+w.value.FP)
}
} else if a.Build != "" && h.have.Build != a.Build {
if a.Build != "" && h.have.Build != a.Build {
waits = append(waits, "the node has not applied "+a.Build)
}
}
@@ -651,13 +509,10 @@ func (m *Model) snapshot() Snapshot {
rows = append(rows, r)
}
for key, h := range m.haves {
node, module, _, perr := delivery.ParseHaveKey(key)
if perr != nil {
continue
}
if a, ok := m.assigns[node+"."+module]; ok && a.open() {
if a, ok := m.assigns[key]; ok && a.open() {
continue
}
node, module, _, _ := delivery.ParseHaveKey(key)
r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err}
if h.err == "" && h.have.Phase == delivery.PhaseRemoved {
// The node-engine says it took the module away: plainly removed, not left behind.
@@ -665,9 +520,6 @@ func (m *Model) snapshot() Snapshot {
}
if h.err == "" {
r.Have, r.Phase, r.Health, r.Detail = h.have.Build, string(h.have.Phase), string(h.have.Health), h.have.Detail
if r.Have == "" {
r.Have = h.have.FP
}
}
rows = append(rows, r)
}
@@ -733,9 +585,9 @@ func (m *Model) snapshot() Snapshot {
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerAsksBucket, Key: key, Error: a.err})
}
}
for key, w := range m.wants {
if w.err != "" {
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerWantsBucket, Key: key, Error: w.err})
for key, d := range m.diffs {
if d.err != "" {
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerDifferencesBucket, Key: key, Error: d.err})
}
}
sort.Slice(offline, func(i, j int) bool {
@@ -749,7 +601,7 @@ func (m *Model) snapshot() Snapshot {
return unreadable[i].Bucket+"."+unreadable[i].Key < unreadable[j].Bucket+"."+unreadable[j].Key
})
return Snapshot{Rows: rows, Nodes: nodes, Offline: offline, Stuck: stuck, Unreadable: unreadable,
Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Wants: m.wantRows(), Source: m.sourceRow()}
Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Differences: m.differenceRows()}
}
// groupRows are the readable candidate groups, ordered by group, their pull requests by repository and number.
@@ -988,38 +840,20 @@ func (m *Model) Answer(askKey, option, words string) error {
return w.Put(key, body)
}
// wantRows are the readable wants beside the haves of their nodes, ordered by node and module.
func (m *Model) wantRows() []WantRow {
out := []WantRow{}
for key, w := range m.wants {
if w.err != "" {
// differenceRows are the readable differences, ordered by node and module.
func (m *Model) differenceRows() []DifferenceRow {
out := []DifferenceRow{}
for key, d := range m.diffs {
if d.err != "" {
continue
}
node, module := splitKey(key)
v := w.value
row := WantRow{Node: node, Module: module, Want: v.FP, Newest: v.Newest, Rule: v.Rule, Words: v.Words}
h, has := m.haves[key]
if has && h.err == "" {
row.Have, row.Phase, row.Health = h.have.FP, string(h.have.Phase), string(h.have.Health)
node, module, err := delivery.ParseDifferenceKey(key)
if err != nil {
continue
}
if ckey, err := delivery.ComposedKey(node, module, v.FP); err == nil && !v.Removed {
if c, ok := m.composed[ckey]; ok && c.err == "" {
row.Judgement, row.Judged = string(c.value.Judgement), c.value.Words
}
}
switch {
case v.Removed:
row.State = WantRemoved
case v.Rule != 0 || v.Newest != "":
row.State = WantHeldBack
case row.Have == v.FP && h.have.Health == delivery.HealthFailed:
row.State = WantFailed
case row.Have == v.FP && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK:
row.State = WantApplied
default:
row.State = WantPending
}
out = append(out, row)
v := d.value
out = append(out, DifferenceRow{Node: node, Module: module, Want: v.Want, WantRemoved: v.WantRemoved, Sent: v.Sent, SentBuild: v.SentBuild,
Rule: v.Rule, Unexplained: v.Unexplained, Words: v.Words})
}
sort.Slice(out, func(i, j int) bool {
if out[i].Node != out[j].Node {
@@ -1029,257 +863,3 @@ func (m *Model) wantRows() []WantRow {
})
return out
}
// sourceRow derives the state of the source revision the gate holds (ADR 0429 §1; recorded by no part): delivered when every want
// equals its module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is
// assigned to has a want. It is nil when the gate holds no readable source.
func (m *Model) sourceRow() *SourceRow {
if m.source == nil || m.source.err != "" {
return nil
}
waiting := []string{}
short := func(fp string) string {
fp = strings.TrimPrefix(fp, delivery.SourcePrefix2)
if len(fp) > 12 {
fp = fp[:12]
}
return fp
}
looked := 0
for key, w := range m.wants {
node, module := splitKey(key)
mod, ok := m.modules[module]
if w.err != "" || w.value.Removed || !ok || mod.err != "" || mod.value.FP2 == "" {
continue
}
looked++
h := m.haves[key]
switch {
case w.value.FP != mod.value.FP2:
waiting = append(waiting, fmt.Sprintf("%s on %s wants %s, not the module's %s", module, node, short(w.value.FP), short(mod.value.FP2)))
case h.err != "" || h.have.FP != w.value.FP:
waiting = append(waiting, fmt.Sprintf("%s on %s does not yet run %s", module, node, short(w.value.FP)))
case h.have.Health == delivery.HealthFailed:
waiting = append(waiting, fmt.Sprintf("%s on %s runs %s and its health check failed", module, node, short(w.value.FP)))
case h.have.Phase != delivery.PhaseRunning || h.have.Health != delivery.HealthOK:
waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet running and healthy", module, node, short(w.value.FP)))
}
}
for key, a := range m.assigns {
if !a.open() {
continue
}
wk, werr := delivery.HaveKey(a.Node, a.Module)
if werr != nil {
wk = key
}
if _, ok := m.wants[wk]; ok {
continue
}
if mod, ok := m.modules[a.Module]; ok && mod.err == "" && mod.value.FP2 != "" {
waiting = append(waiting, fmt.Sprintf("%s on %s has no want yet", a.Module, a.Node))
}
}
sort.Strings(waiting)
row := &SourceRow{Revision: m.source.revision, State: SourceDelivered, Wants: looked}
if len(waiting) > 0 {
row.State, row.Waiting = SourceNotDelivered, waiting
}
return row
}
// splitKey is the node and the module (unescaped, issue 1150) of a model key, which is the have bucket's key form.
func splitKey(key string) (node, module string) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
node, module, _ = strings.Cut(key, ".")
}
return node, module
}
// The events of a module in motion (issue 1906 D10, for the messenger, ADR 0308): with no plan to say it, the view says that a
// module's want on a node differs from its have (moving), that the have reached the want running and healthy (settled), or that it
// is held or failed or stopped moving past the watchman's bound (stuck). Each is said once for a node, a module and a composition.
const (
MotionMoving = "moving"
MotionSettled = "settled"
MotionStuck = "stuck"
// MotionBoundSeconds is how long a node may take to apply a want before the watchman says it stuck (watch.StepBound).
MotionBoundSeconds = 1800
)
// MotionBody is the body of the three events. Outcome and Words are those of settled and stuck: settled is "running"; stuck is
// "failed" (a health check failed) or "stopped" (the module is held, or the watchman says the want stopped moving past its bound),
// with the have's own words or the watchman's in Words.
type MotionBody struct {
Node string `json:"node"`
Module string `json:"module"`
FromFP string `json:"from_fp"`
ToFP string `json:"to_fp"`
Composed string `json:"composed"`
Since time.Time `json:"since"`
BoundSeconds int `json:"bound_seconds"`
// At is when the have or the want last changed, the time the event is about.
At time.Time `json:"at"`
Outcome string `json:"outcome,omitempty"`
Words string `json:"words,omitempty"`
}
// MotionEvent is an event to say: its type (moving, settled or stuck), its id (the bus's dedup id and the event id header, one for
// a type, a node, a module and a composition) and its body.
type MotionEvent struct {
Type string
ID string
Body MotionBody
}
// SetMotionEvent sets where the events of modules in motion go; nothing is said until it is set.
func (m *Model) SetMotionEvent(say func(MotionEvent)) {
m.mu.Lock()
defer m.mu.Unlock()
m.motionEvent = say
}
// motionNow is the events to say now about the node and module the change touched: a want, a have, or the watchman's stuck key of a
// want.
func (m *Model) motionNow(e Entry) []MotionEvent {
if m.motionEvent == nil || !m.motionReady() {
return nil
}
var hk string
switch e.Bucket {
case delivery.AssignerWantsBucket:
node, module, err := delivery.ParseWantKey(e.Key)
if err != nil {
return nil
}
hk, _ = delivery.HaveKey(node, module)
case delivery.HaveBucket:
if _, _, isNode, err := delivery.ParseHaveKey(e.Key); err != nil || isNode {
return nil
}
hk = e.Key
case delivery.StuckBucket:
st, ok := m.stuck[e.Key]
if !ok || st.err != "" || st.value.Bucket != delivery.AssignerWantsBucket {
return nil
}
node, module, err := delivery.ParseWantKey(st.value.Key)
if err != nil {
return nil
}
hk, _ = delivery.HaveKey(node, module)
default:
return nil
}
return m.motionOf(hk)
}
// motionReady says whether the view has read what stands in the buckets motion is derived from (the wants, the haves and the
// watchman's stuck keys): before, a want read ahead of its have would be said moving though the node runs it.
func (m *Model) motionReady() bool {
for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
if !m.initial[b] {
return false
}
}
return true
}
// Initial tells the model that the watch of a bucket has handed over every key that stood at its start. When the buckets motion is
// derived from have all done so, the events of the modules in motion at that moment are said (the ids deduplicate what a restart
// of the view says again).
func (m *Model) Initial(bucket string) {
m.mu.Lock()
was := m.motionReady()
m.initial[bucket] = true
var out []MotionEvent
if !was && m.motionReady() && m.motionEvent != nil {
keys := make([]string, 0, len(m.wants))
for k := range m.wants {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
out = append(out, m.motionOf(k)...)
}
}
say := m.motionEvent
m.mu.Unlock()
for _, ev := range out {
say(ev)
}
}
// motionOf is the events to say now about the node and module of a model key, from their want, have and stuck key.
func (m *Model) motionOf(hk string) []MotionEvent {
w, ok := m.wants[hk]
if !ok || w.err != "" || w.value.Removed || w.value.FP == "" {
return nil
}
node, module := splitKey(hk)
h, has := m.haves[hk]
has = has && h.err == ""
body := MotionBody{Node: node, Module: module, ToFP: w.value.FP, Composed: w.value.Composed, Since: w.at, BoundSeconds: MotionBoundSeconds}
body.At = w.at
if has {
body.FromFP = h.have.FP
if h.at.After(body.At) {
body.At = h.at
}
}
id := func(kind string) string {
return kind + "." + node + "." + module + "." + strings.TrimPrefix(w.value.Composed, "sha256:")
}
applied := has && h.have.FP == w.value.FP && (h.have.Composed == "" || h.have.Composed == w.value.Composed)
var out []MotionEvent
say := func(kind string, b MotionBody) {
if m.told[id(kind)] {
return
}
m.told[id(kind)] = true
out = append(out, MotionEvent{Type: kind, ID: id(kind), Body: b})
}
switch {
case applied && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK:
if m.told[id(MotionMoving)] {
b := body
b.Outcome = "running"
say(MotionSettled, b)
}
return out
case !m.told[id(MotionMoving)]:
say(MotionMoving, body)
}
stop := body
switch {
case has && h.have.Phase == delivery.PhaseHeld:
stop.Outcome, stop.Words = "stopped", h.have.Detail
case has && (h.have.Health == delivery.HealthFailed || h.have.Phase == delivery.PhaseFailed) && h.have.FP == w.value.FP:
stop.Outcome, stop.Words = "failed", h.have.Detail
default:
if s, ok := m.stuckWant(w.value, node, module); ok {
stop.Outcome, stop.Words = "stopped", s
} else {
return out
}
}
say(MotionStuck, stop)
return out
}
// stuckWant is the watchman's words for the want of module on node, when it says the want stopped moving and not yet moved.
func (m *Model) stuckWant(_ delivery.Want, node, module string) (string, bool) {
wk, err := delivery.WantKey(node, module)
if err != nil {
return "", false
}
sk, err := delivery.StuckKey(delivery.AssignerWantsBucket, wk)
if err != nil {
return "", false
}
if st, ok := m.stuck[sk]; ok && st.err == "" && !st.value.Moved {
return st.value.Waited, true
}
return "", false
}
modules/mesh-delivery-view/internal/view/wants_test.go
-382
View File
@@ -1,382 +0,0 @@
package view
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const (
fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
)
func wantE(t *testing.T, node, module string, w delivery.Want) Entry {
t.Helper()
key, err := delivery.WantKey(node, module)
if err != nil {
t.Fatal(err)
}
if !w.Removed {
w.Composed = "sha256:" + strings.Repeat("c", 64)
}
v, err := delivery.EncodeWant(w)
if err != nil {
t.Fatal(err)
}
return Entry{Bucket: delivery.AssignerWantsBucket, Key: key, Value: v, At: t0}
}
func haveFP(t *testing.T, node, module, fp string, health delivery.Health) Entry {
t.Helper()
b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: health})
if err != nil {
t.Fatal(err)
}
key, err := delivery.HaveKey(node, module)
if err != nil {
t.Fatal(err)
}
return Entry{Bucket: delivery.HaveBucket, Key: key, Value: b, At: t0}
}
func moduleE(t *testing.T, module, fp string) Entry {
t.Helper()
v, err := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/" + module, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
if err != nil {
t.Fatal(err)
}
return Entry{Bucket: delivery.ModulesBucket, Key: module, Value: v, At: t0}
}
func sourceE(t *testing.T, revision uint64) Entry {
t.Helper()
v, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}})
if err != nil {
t.Fatal(err)
}
return Entry{Bucket: delivery.SourceBucket, Key: delivery.SourceMainKey, Value: v, At: t0, Revision: revision}
}
// 1906/D7 AC7: every want is a row beside the have of its node, with where it stands.
func TestEveryWantIsARowAgainstItsHave(t *testing.T) {
m := New()
fill(m,
wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK),
wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK),
wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed),
wantE(t, "g14", "mail", delivery.Want{FP: fpOld, Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"}),
wantE(t, "g14", "old", delivery.Want{Removed: true}),
wantE(t, "novox", "web", delivery.Want{FP: fpNew}),
)
got := map[string]WantRow{}
for _, r := range m.Snapshot().Wants {
got[r.Node+"."+r.Module] = r
}
for key, state := range map[string]string{
"ace.web": WantApplied, "ace.mail": WantPending, "g14.web": WantFailed, "g14.mail": WantHeldBack, "g14.old": WantRemoved, "novox.web": WantPending,
} {
if got[key].State != state {
t.Errorf("%s = %+v, want state %s", key, got[key], state)
}
}
if r := got["g14.mail"]; r.Rule != delivery.RuleCanary || r.Newest != fpNew || r.Words == "" {
t.Errorf("the held-back row keeps its rule and words: %+v", r)
}
if r := got["ace.mail"]; r.Want != fpNew || r.Have != fpOld {
t.Errorf("a pending row shows both: %+v", r)
}
// a want taken away leaves the list
fill(m, Entry{Bucket: delivery.AssignerWantsBucket, Key: "want.novox.web", Deleted: true})
if n := len(m.Snapshot().Wants); n != 5 {
t.Errorf("wants = %d after a delete, want 5", n)
}
}
// 1906/D7 AC7: a source revision is delivered when every want equals its module's fingerprint and every have equals its
// want, healthy; one failed have shows it not delivered and names the module.
func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) {
m := New()
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew),
assign("ace", "web", "b-1", ""), assign("g14", "web", "b-1", ""),
wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK),
wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK),
wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpNew, delivery.HealthOK))
s := m.Snapshot().Source
if s == nil || s.Revision != 14355 || s.State != SourceDelivered || len(s.Waiting) != 0 {
t.Fatalf("source = %+v, want revision 14355 delivered", s)
}
fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthFailed))
s = m.Snapshot().Source
if s.State != SourceNotDelivered || len(s.Waiting) != 1 || !strings.Contains(s.Waiting[0], "web") || !strings.Contains(s.Waiting[0], "g14") {
t.Errorf("a failed have: %+v", s)
}
fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "mail", delivery.Want{FP: fpOld}))
s = m.Snapshot().Source
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") {
t.Errorf("a want that is not the module's fingerprint: %+v", s)
}
fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), assign("novox", "web", "b-1", ""))
s = m.Snapshot().Source
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") {
t.Errorf("a node assigned the module with no want: %+v", s)
}
}
func TestNoSourceRevisionHeldMeansNoStateIsDerived(t *testing.T) {
if s := New().Snapshot().Source; s != nil {
t.Errorf("source = %+v", s)
}
}
func TestTheWantsPageSaysWantsAgainstHaves(t *testing.T) {
m := New()
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
text := wantText(m.Snapshot())
if len(text) != 1 || text[0][0] != "ace" || text[0][1] != "web" || text[0][2] != fpNew || text[0][3] != fpOld || text[0][4] != WantPending || text[0][5] != "none" {
t.Errorf("text = %v", text)
}
}
// 1906/D7 (ADR 0539 §14): the controller's judgement of the move a want names is shown beside it, with its words.
func TestTheJudgementOfTheMoveIsShownBesideTheWant(t *testing.T) {
m := New()
key, err := delivery.ComposedKey("ace", "web", fpNew)
if err != nil {
t.Fatal(err)
}
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"})
if err != nil {
t.Fatal(err)
}
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.ControllerComposedBucket, Key: key, Value: v, At: t0})
rows := m.Snapshot().Wants
if len(rows) != 1 || rows[0].Judgement != string(delivery.JudgedCannot) || !strings.Contains(rows[0].Judged, "gone") {
t.Fatalf("rows = %+v", rows)
}
text := wantText(m.Snapshot())
if len(text[0]) != 7 || text[0][5] != "cannot-judge" {
t.Errorf("text = %v", text)
}
}
// 1906/D9 (asked by the lead for issue 1946): the view says, once for each source revision, that it is delivered: the event the
// issue tracker derives "fix delivered" from, in place of the gate's states that go in release 6 (ADR 0539 §17).
func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) {
m := New()
var said []DeliveredEvent
m.SetDeliveredEvent(func(e DeliveredEvent) { said = append(said, e) })
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), assign("ace", "web", "b-1", ""),
wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
if len(said) != 0 {
t.Fatalf("said %v before any have", said)
}
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed))
if len(said) != 0 {
t.Fatalf("said %v for a failed have", said)
}
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
if len(said) != 1 || said[0].Revision != 14355 || said[0].Group != "g" || said[0].Heads["novox/web"] == "" {
t.Fatalf("said = %+v, want revision 14355 once with its group and heads", said)
}
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
if len(said) != 1 {
t.Errorf("said again for the same revision: %v", said)
}
// a newer revision is delivered when its wants and haves are; a revision that goes bad and good again is not said twice
fill(m, sourceE(t, 14356))
if len(said) != 2 || said[1].Revision != 14356 {
t.Errorf("the next revision: %v", said)
}
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed), haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
if len(said) != 2 {
t.Errorf("said again after a have failed and recovered: %v", said)
}
}
// Review round 1 (view notes): the controller no longer writes the build of an assignment (C6), so a row reads the want and the
// have: the row shows the fingerprint wanted and waits while the node runs another.
func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) {
m := New()
fill(m, assign("ace", "web", "", ""), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
r := find(t, m.Snapshot(), "ace", "web")
if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") {
t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r)
}
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
if r := find(t, m.Snapshot(), "ace", "web"); r.Waits != "" || r.Have != fpNew {
t.Errorf("row = %+v, want no wait once the have names the want's fingerprint", r)
}
}
// Review round 1 (view notes, issue 1150): a module whose name holds a dot has its have under the escaped key; the row finds it,
// and a have nothing is assigned to keeps the module's name.
func TestADottedModuleNameIsFoundAndKept(t *testing.T) {
m := New()
fill(m, assign("ace", "novox.be", "", ""), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK),
haveFP(t, "ace", "old.name", fpNew, delivery.HealthOK))
s := m.Snapshot()
if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Waits != "" {
t.Errorf("the dotted module's row = %+v", r)
}
if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind {
t.Errorf("the left-behind dotted module = %+v", r)
}
if rows := s.Wants; len(rows) != 1 || rows[0].Module != "novox.be" || rows[0].State != WantApplied {
t.Errorf("wants = %+v", rows)
}
}
// 1906/D10 (asked by the lead for the messenger, ADR 0308): with no plans to say that a module is in motion, the view says it: `moving`
// when a module's want on a node differs from its have, `settled` when the have reaches the want running and healthy, `stuck` when it
// is held or failed or the watchman says it stopped moving; each once for a node, a module and a composition.
func motionLog(m *Model) *[]MotionEvent {
var said []MotionEvent
m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) })
for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
m.Initial(b)
}
return &said
}
func types(said []MotionEvent) string {
var out []string
for _, e := range said {
out = append(out, e.Type)
}
return strings.Join(out, ",")
}
func TestAModuleInMotionIsSaidMovingThenSettledOnceEach(t *testing.T) {
m := New()
said := motionLog(m)
fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
if len(*said) != 0 {
t.Fatalf("said %v for a have with no want", types(*said))
}
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
if types(*said) != "moving" {
t.Fatalf("said %q, want moving", types(*said))
}
mv := (*said)[0]
if mv.Body.Node != "ace" || mv.Body.Module != "web" || mv.Body.FromFP != fpOld || mv.Body.ToFP != fpNew || mv.Body.Composed != "sha256:"+strings.Repeat("c", 64) ||
mv.Body.Since.IsZero() || mv.Body.At.IsZero() || mv.Body.BoundSeconds != 1800 || mv.ID != "moving.ace.web."+strings.Repeat("c", 64) {
t.Errorf("moving = %+v", mv)
}
fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
if types(*said) != "moving" {
t.Errorf("said %q after the same state again", types(*said))
}
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
if types(*said) != "moving,settled" || (*said)[1].Body.Outcome != "running" {
t.Errorf("said %q, want settled with its outcome", types(*said))
}
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
if types(*said) != "moving,settled" {
t.Errorf("settled said twice: %q", types(*said))
}
}
func TestAModuleHeldOrFailedIsSaidStuckWithItsWords(t *testing.T) {
m := New()
said := motionLog(m)
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
b, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0})
if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" || (*said)[1].Body.Words != "no space left" {
t.Fatalf("said %v: %+v", types(*said), *said)
}
fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0})
if types(*said) != "moving,stuck" {
t.Errorf("stuck said twice: %q", types(*said))
}
// failed on the wanted fingerprint
m2 := New()
said2 := motionLog(m2)
fill(m2, wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed))
if types(*said2) != "moving,stuck" || (*said2)[1].Body.Outcome != "failed" {
t.Errorf("failed: said %q", types(*said2))
}
}
func TestAWantTheWatchmanSaysStoppedMovingIsSaidStuckPastItsBound(t *testing.T) {
m := New()
said := motionLog(m)
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, "want.ace.web")
v, err := delivery.EncodeStuck(delivery.Stuck{Bucket: delivery.AssignerWantsBucket, Key: "want.ace.web", Waited: "the node agent of ace to run web", Since: t0})
if err != nil {
t.Fatal(err)
}
fill(m, Entry{Bucket: delivery.StuckBucket, Key: sk, Value: v, At: t0})
if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" {
t.Errorf("said %q: %+v", types(*said), *said)
}
}
func TestANodeThatSettlesWithoutBeingSeenMovingSaysNothing(t *testing.T) {
m := New()
said := motionLog(m)
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
if len(*said) != 0 {
t.Errorf("said %v for a module that never moved", types(*said))
}
}
// Nothing is said of a module in motion before the buckets have been read: a want read ahead of its have is not a node behind. What is
// in motion when they have been read is said then, and a module that settles after is said settled.
func TestMotionIsSaidOnlyOnceTheBucketsHaveBeenRead(t *testing.T) {
m := New()
var said []MotionEvent
m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) })
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), wantE(t, "ace", "mail", delivery.Want{FP: fpNew}))
m.Initial(delivery.AssignerWantsBucket)
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK))
m.Initial(delivery.HaveBucket)
if len(said) != 0 {
t.Fatalf("said %v before the stuck keys were read", types(said))
}
m.Initial(delivery.StuckBucket)
if len(said) != 1 || said[0].Type != MotionMoving || said[0].Body.Module != "mail" {
t.Fatalf("said %v: %+v, want only mail moving (web is already there)", types(said), said)
}
fill(m, haveFP(t, "ace", "mail", fpNew, delivery.HealthOK))
if types(said) != "moving,settled" {
t.Errorf("said %q", types(said))
}
}
// assigner_wants lists a node's own want (want.<node>.node, ADR 0539 clause 6) among the wants, and never as unreadable:
// on 2026-10-25 every node's own want was listed unreadable, though its value was what the assigner writes (issue 1945).
func TestTheWantsListANodesOwnWant(t *testing.T) {
own, err := delivery.WantNodeKey("ace")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: "sha256:" + strings.Repeat("d", 64)})
if err != nil {
t.Fatal(err)
}
moduleKey, _ := delivery.WantKey("ace", "web")
moduleValue, _ := delivery.EncodeWant(delivery.Want{FP: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)})
state := map[string][]byte{own: value, moduleKey: moduleValue}
r := Reader{WantsState: StateRead{
Keys: func() ([]string, error) { return []string{own, moduleKey}, nil },
Get: func(k string) (json.RawMessage, error) { return state[k], nil },
}}
got, err := r.Wants("", "")
if err != nil {
t.Fatal(err)
}
if len(got.Unreadable) != 0 || got.Wants[own] == nil || got.Wants[moduleKey] == nil {
t.Fatalf("the wants: %v, unreadable %v", got.Wants, got.Unreadable)
}
if got, _ := r.Wants("g14", ""); len(got.Wants) != 0 {
t.Errorf("another node's wants: %v", got.Wants)
}
if got, _ := r.Wants("ace", "web"); got.Wants[own] != nil {
t.Errorf("the node's own want was listed under the module web")
}
}
modules/mesh-delivery-view/manifest_test.go
+3 -8
View File
@@ -50,7 +50,6 @@ type manifest struct {
Reads []string `json:"reads"`
Uses []string `json:"uses"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
OwnSecrets map[string]string `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
Resources []map[string]any `json:"resources"`
@@ -84,21 +83,17 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
if m.Module != "mesh-delivery-view" {
t.Errorf("module = %q", m.Module)
}
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
got := slices.Clone(m.Reads)
slices.Sort(got)
if !reflect.DeepEqual(got, want) {
t.Errorf("reads = %v, want exactly %v", got, want)
}
for _, k := range []string{"consumes", "provides", "claims"} {
for _, k := range []string{"emits", "consumes", "provides", "claims"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the view writes only its answers", k)
}
}
// The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves.
if !reflect.DeepEqual(m.Emits, []string{"delivered", "moving", "settled", "stuck"}) {
t.Errorf("emits = %v, want exactly delivered, moving, settled and stuck (1906/D9, D10)", m.Emits)
}
// The view uses no build seat: it reads their workers by the broker's named grant and publishes no ask (ADR 0525 §2).
if len(m.Uses) != 0 {
t.Errorf("uses = %v, want none", m.Uses)
@@ -115,7 +110,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
}
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves"}; !reflect.DeepEqual(m.Tools, want) {
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
for _, a := range m.Build.Artifacts {
modules/mesh-delivery-view/module.json
+1 -14
View File
@@ -9,17 +9,7 @@
"indexer_candidate",
"indexer_candidates",
"builder_build",
"builder_builds",
"assigner_wants",
"assigner_want",
"assigner_differences",
"node_haves"
],
"emits": [
"delivered",
"moving",
"settled",
"stuck"
"builder_builds"
],
"state": [
{ "name": "answers", "history": 5 }
@@ -27,13 +17,10 @@
"reads": [
"mesh-controller.assignments",
"mesh-controller.builds",
"mesh-controller.composed",
"mesh-delivery-assigner.asks",
"mesh-delivery-assigner.differences",
"mesh-delivery-assigner.wants",
"mesh-delivery-builder.builds",
"mesh-delivery-gate.asks",
"mesh-delivery-gate.source",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers",
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go
-136
View File
@@ -9,7 +9,6 @@ import (
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -100,8 +99,6 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
{Bucket: delivery.StuckBucket, History: 20},
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
{Bucket: delivery.AssignerWantsBucket, History: 20},
{Bucket: delivery.HaveBucket, History: 20},
} {
kv, err := js.CreateKeyValue(ctx, c)
if err != nil {
@@ -268,8 +265,6 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
{Bucket: delivery.StuckBucket, History: 20},
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
{Bucket: delivery.AssignerWantsBucket, History: 20},
{Bucket: delivery.HaveBucket, History: 20},
} {
kv, err := js.CreateKeyValue(ctx, c)
if err != nil {
@@ -307,134 +302,3 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
}
t.Fatalf("no pulse for the ask running before the start (%s)", pk)
}
// TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt (1906/D6, ADR 0539 §19): started whole, the watchman mirrors a want of
// the assigner as a pulse owed by the node's agent, one that stood before its start as well, and the node-engine's have that
// names the want's fingerprint deletes the pulse and writes no stuck key.
func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "w")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
conf := filepath.Join(dir, "bus.conf")
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatal(err)
}
opts.NoLog, opts.NoSigs = true, true
srv, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(ctl.Close)
js, err := jetstream.New(ctl)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
kvs := map[string]jetstream.KeyValue{}
for _, c := range []jetstream.KeyValueConfig{
{Bucket: delivery.PresenceBucket, History: 20},
{Bucket: delivery.StuckBucket, History: 20},
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
{Bucket: delivery.AssignerWantsBucket, History: 20},
{Bucket: delivery.HaveBucket, History: 20},
} {
kv, err := js.CreateKeyValue(ctx, c)
if err != nil {
t.Fatal(err)
}
kvs[c.Bucket] = kv
}
fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
putWant := func(node, module, fp string) {
t.Helper()
body, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.WantKey(node, module)
if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, key, body); err != nil {
t.Fatal(err)
}
}
putHave := func(node, module, fp string) {
t.Helper()
body, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveKey(node, module)
if _, err := kvs[delivery.HaveBucket].Put(ctx, key, body); err != nil {
t.Fatal(err)
}
}
putHave("n2", "web", fpOld)
putWant("n2", "web", fpNew) // stood before the start
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-watchman", User: "n1.mesh-delivery-watchman", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
pulse := func(node, module string) string {
wk, _ := delivery.WantKey(node, module)
pk, err := delivery.PulseKey(delivery.AssignerWantsBucket, wk)
if err != nil {
t.Fatal(err)
}
return pk
}
until := func(what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if ok() {
return
}
}
t.Fatalf("never saw: %s", what)
}
has := func(key string) bool { _, err := kvs[delivery.PulseBucket].Get(ctx, key); return err == nil }
until("a pulse for the want that stood before the start", func() bool { return has(pulse("n2", "web")) })
putWant("n2", "mail", fpNew)
until("a pulse for a want written after the start", func() bool { return has(pulse("n2", "mail")) })
e, err := kvs[delivery.PulseBucket].Get(ctx, pulse("n2", "web"))
if err != nil {
t.Fatal(err)
}
if p, err := delivery.DecodePulse(e.Value()); err != nil || !strings.Contains(p.Owes, "n2") {
t.Errorf("the pulse = %+v, %v; want it owed by the agent of n2", p, err)
}
putHave("n2", "web", fpNew)
until("the pulse of the want its have names deleted", func() bool { return !has(pulse("n2", "web")) })
if !has(pulse("n2", "mail")) {
t.Errorf("the pulse of another want ended with web's have")
}
for _, m := range []string{"web"} {
wk, _ := delivery.WantKey("n2", m)
sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, wk)
if _, err := kvs[delivery.StuckBucket].Get(ctx, sk); err == nil {
t.Errorf("a stuck key was written for the want of %s that was applied", m)
}
}
}
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go
+2 -19
View File
@@ -229,12 +229,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
return kv, nil
}
var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue }
var kvs struct{ presence, pulse, stuck jetstream.KeyValue }
for _, b := range []struct {
name string
into *jetstream.KeyValue
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck},
{delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} {
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} {
if *b.into, err = bucket(b.name); err != nil {
return err
}
@@ -281,16 +280,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent
// until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse.
haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
if err != nil {
return err
}
wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
if err != nil {
return err
}
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
// keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same.
taken := map[string]jetstream.KeyWatcher{}
@@ -348,12 +337,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go func() {
errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) })
}()
go func() {
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
}()
go func() {
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
}()
for lang, sw := range taken {
lang, sw := lang, sw
go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }()
modules/mesh-delivery-watchman/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-watchman/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-watchman/internal/watch/mirror.go
-96
View File
@@ -1,9 +1,6 @@
package watch
import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -55,96 +52,3 @@ func (w *Watchman) Step(language, key string, value []byte, op Op) {
}
w.mirrorWithin(w.stepBound, bucket, id, describe("the ask %s to end on %s", id, node), node, since, 0)
}
// A want is a move too (novox/hq ADR 0539 §19, issue 1906): the assigner writes `want.<node>.<module>` in its wants bucket,
// and the node's agent owes the node-engine's have naming the want's fingerprint. The pulse is the pulse of that bucket under
// the want's key, owed by the node's agent, and it begins when the assigner wrote the want. The have naming the fingerprint
// ends it; a want that is removed, or a key deleted, ends it as well. Nothing renews it: the bound is how long a node may
// take to apply a want, the same as a build step's (StepBound). Keys of that bucket that are not a want of a module (a hold,
// and the node's own key) are left.
// Want is a change of a key of the assigner's wants bucket, read from the watch (at the start the keys that stand, then each
// change), at the time the entry was written.
func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
node, module, err := delivery.ParseWantKey(key)
if err != nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
switch op {
case Expired:
return
case Delete:
delete(w.wants, key)
w.moved(delivery.AssignerWantsBucket, key)
return
}
want, err := delivery.DecodeWant(value)
if err != nil {
w.say("the want %s cannot be read, and is left: %v", key, err)
return
}
if want.Removed {
delete(w.wants, key)
w.moved(delivery.AssignerWantsBucket, key)
return
}
w.wants[key] = want.FP
if w.have(node, module) == want.FP {
w.moved(delivery.AssignerWantsBucket, key)
return
}
if at.IsZero() {
at = w.now()
}
w.mirrorWithin(w.stepBound, delivery.AssignerWantsBucket, key,
describe("the node agent of %s to run %s at %s", node, module, shortFP(want.FP)), want.FP, at, 0)
}
// Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the
// fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left.
func (w *Watchman) Have(key string, value []byte, op Op) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
return
}
w.mu.Lock()
defer w.mu.Unlock()
switch op {
case Expired:
return
case Delete:
delete(w.haves, key)
return
}
have, err := delivery.DecodeHave(value)
if err != nil {
w.say("the have %s cannot be read, and is left: %v", key, err)
return
}
w.haves[key] = have.FP
wk, err := delivery.WantKey(node, module)
if err != nil {
return
}
if fp, ok := w.wants[wk]; ok && fp == have.FP {
w.moved(delivery.AssignerWantsBucket, wk)
}
}
func (w *Watchman) have(node, module string) string {
key, err := delivery.HaveKey(node, module)
if err != nil {
return ""
}
return w.haves[key]
}
func shortFP(fp string) string {
hex := strings.TrimPrefix(fp, delivery.SourcePrefix2)
if len(hex) > 12 {
hex = hex[:12]
}
return hex
}
modules/mesh-delivery-watchman/internal/watch/watch.go
+1 -3
View File
@@ -57,8 +57,6 @@ type Watchman struct {
presence map[string]presence
pulses map[string]*pulse // live pulses, by pulse key
stuck map[string]delivery.Stuck
wants map[string]string // the fingerprint of each want that stands, by want key
haves map[string]string // the fingerprint each node runs of a module, by have key
bound time.Duration
// stepBound is how long a build step may run once a node took the ask; a test shortens it.
stepBound time.Duration
@@ -67,7 +65,7 @@ type Watchman struct {
// New is a watchman writing to out, reading the time from now.
func New(out Out, now func() time.Time) *Watchman {
return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{},
pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound}
pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, bound: delivery.PulseMoveBound, stepBound: StepBound}
}
// Logf sets where a write that failed is said; it is silent until set.
modules/mesh-delivery-watchman/internal/watch/watch_test.go
-89
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
"errors"
"slices"
"strings"
"testing"
"time"
@@ -532,91 +531,3 @@ func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908
t.Errorf("stuck %v pulses %v", o2.stuck, o2.pulses)
}
}
// 1906/D6 (ADR 0539 §19): a want is a pulse owed by the node's agent from when the assigner wrote it, until the have names
// its fingerprint.
func wantValue(t *testing.T, fp string, removed bool) []byte {
t.Helper()
w := delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed}
if removed {
w.Composed = ""
}
b, err := delivery.EncodeWant(w)
if err != nil {
t.Fatal(err)
}
return b
}
func haveValue(t *testing.T, fp string) []byte {
t.Helper()
b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
if err != nil {
t.Fatal(err)
}
return b
}
var (
fpOld = delivery.SourcePrefix2 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
fpNew = delivery.SourcePrefix2 + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
)
func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) {
w, o, c := setup()
c.t = t0.Add(10 * time.Second)
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")
p, ok := o.pulses[pk]
if !ok || p.Key != "want.ace.web" || !strings.Contains(p.Owes, "ace") || !strings.Contains(p.Owes, "web") {
t.Fatalf("pulses = %v", o.pulses)
}
w.Have("ace.web", haveValue(t, fpOld), Put) // the node still runs the old one
if _, ok := o.pulses[pk]; !ok {
t.Fatalf("a have of another fingerprint ended the pulse")
}
w.Have("ace.web", haveValue(t, fpNew), Put)
if _, ok := o.pulses[pk]; ok {
t.Errorf("the have naming the want's fingerprint left the pulse standing: %v", o.pulses)
}
if len(o.stuck) != 0 {
t.Errorf("stuck = %v", o.stuck)
}
}
func TestAHaveThatCameBeforeTheWantEndsItAtOnceAndARemovedWantHasNoPulse(t *testing.T) {
w, o, c := setup()
c.t = t0.Add(time.Second)
w.Have("ace.web", haveValue(t, fpNew), Put)
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
w.Want("want.ace.mail", wantValue(t, "", true), Put, t0)
if len(o.pulses) != 0 {
t.Errorf("pulses = %v", o.pulses)
}
// the want moving on to a new fingerprint makes a new pulse; the key `want.ace.node` and a held key are left
w.Want("want.ace.web", wantValue(t, fpOld, false), Put, t0.Add(time.Second))
w.Want("want.ace.node", []byte(`{}`), Put, t0)
w.Want("held.web.bbbb", []byte(`{}`), Put, t0)
if len(o.pulses) != 1 {
t.Errorf("pulses = %v, want the web want's only", o.pulses)
}
}
func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) {
o, c := newOut(), &clock{t: t0}
w := New(o, c.now)
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")
c.t = t0.Add(31 * time.Minute)
w.Pulse(pk, Expired, c.t)
if s, ok := o.stuck[stuckKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")]; !ok || s.Moved || !s.Since.Equal(t0) {
t.Fatalf("stuck = %v", o.stuck)
}
w2, o2, c2 := setup()
c2.t = t0.Add(time.Second)
w2.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
w2.Want("want.ace.web", nil, Delete, t0)
if len(o2.pulses) != 0 {
t.Errorf("a deleted want left its pulse: %v", o2.pulses)
}
}
modules/mesh-delivery-watchman/manifest_test.go
+1 -1
View File
@@ -81,7 +81,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-delivery-assigner.wants", "node-engine.have"}) {
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps"}) {
t.Errorf("reads = %v", reads)
}
for _, k := range []string{"emits", "context", "items", "consumes", "provides", "claims"} {
modules/mesh-delivery-watchman/module.json
+1 -3
View File
@@ -5,9 +5,7 @@
"reads": [
"mesh-build-go.steps",
"mesh-build-image.steps",
"mesh-build-node.steps",
"mesh-delivery-assigner.wants",
"node-engine.have"
"mesh-build-node.steps"
],
"state": [
{ "name": "presence", "history": 20 },