Compare commits
59
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8a434b8c6 | ||
|
|
5ba6881f9f | ||
|
|
61196c99e6 | ||
|
|
250364a57b | ||
|
|
73779a5dd1 | ||
|
|
ec76ab77c0 | ||
|
|
91d041b62a | ||
|
|
51444a8341 | ||
|
|
dd9138310c | ||
|
|
653da54c0d | ||
|
|
f72f57f6e2 | ||
|
|
975f0ed654 | ||
|
|
706169aa8a | ||
|
|
6d2d50b1e1 | ||
|
|
3d7d65edce | ||
|
|
06fd53e5cb | ||
|
|
81182e9dcd | ||
|
|
f7b23033a2 | ||
|
|
cf26e95691 | ||
|
|
8daa5977f9 | ||
|
|
30ca8a22bc | ||
|
|
97e605edb0 | ||
|
|
7ef796b04b | ||
|
|
ec93585736 | ||
|
|
6b63f13da6 | ||
|
|
2268f94ef2 | ||
|
|
48aaecf556 | ||
|
|
650a06ceab | ||
|
|
3e863884fd | ||
|
|
c21bed20a3 | ||
|
|
aa31e85665 | ||
|
|
36a1fd197f | ||
|
|
9a5b3d4881 | ||
|
|
e263049551 | ||
|
|
0f7bc9f531 | ||
|
|
da31740f00 | ||
|
|
a3815051cc | ||
|
|
8280506cee | ||
|
|
3b00f3890d | ||
|
|
fcac03a4f2 | ||
|
|
14c20b21ee | ||
|
|
333e0cd07d | ||
|
|
03e5951dc5 | ||
|
|
37d0ebba7a | ||
|
|
86fd0bbe96 | ||
|
|
37a5ab8f1c | ||
|
|
55f19283f4 | ||
|
|
e7a3ff1e7e | ||
|
|
f947070b7b | ||
|
|
04c2c60f02 | ||
|
|
0df20b5d77 | ||
|
|
806c8bf0e4 | ||
|
|
12b01badb0 | ||
|
|
7b18696cd5 | ||
|
|
204a59dab4 | ||
|
|
f048e2b443 | ||
|
|
9d2e9dc6c9 | ||
|
|
024a3787da | ||
|
|
4fd6b934c8 |
@@ -5,6 +5,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -17,9 +18,12 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events
|
||||
@@ -73,6 +77,10 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The operator-channel seat's stream of conditions, which the assigner says its conditions on.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The Go build seat's work queue, as the broker derives it from the seat's accepted kinds (delivery.AskStream).
|
||||
seat, _ := delivery.AskStream(delivery.AskGo)
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: seat, Subjects: []string{"mesh.seat.node-build-go.accept.>"},
|
||||
@@ -85,6 +93,14 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
// start runs the program as the node does, through run with its environment and a broker file; it answers the
|
||||
// channel run's error arrives on.
|
||||
func (b *bed) start(t *testing.T) <-chan error {
|
||||
t.Helper()
|
||||
done, _ := b.startStoppable(t)
|
||||
return done
|
||||
}
|
||||
|
||||
// startStoppable is start, and answers the function that stops this one program (a replaced assigner is stopped, then
|
||||
// another started).
|
||||
func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(b.dir, "broker")
|
||||
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"})
|
||||
@@ -94,15 +110,19 @@ func (b *bed) start(t *testing.T) <-chan error {
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
ctx, stop := context.WithCancel(b.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(ctx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() {
|
||||
go func() {
|
||||
done <- run(ctx, func(k string) string { return env[k] }, io.Discard)
|
||||
close(done) // a program that ended on its own (a refused start) is not waited for again at the cleanup
|
||||
}()
|
||||
stopped := func() {
|
||||
stop()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
})
|
||||
return done
|
||||
}
|
||||
t.Cleanup(stopped)
|
||||
return done, stopped
|
||||
}
|
||||
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
|
||||
@@ -159,6 +179,27 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
|
||||
// move to it, in mesh-controller_composed.
|
||||
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
|
||||
t.Helper()
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat(string(d), 64)
|
||||
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
|
||||
if judgement != delivery.JudgedNeither {
|
||||
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
|
||||
}
|
||||
v, err := delivery.EncodeComposedDigest(cd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
return digest
|
||||
}
|
||||
|
||||
// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest.
|
||||
func (b *bed) manifest(t *testing.T, module, body string) string {
|
||||
t.Helper()
|
||||
@@ -240,10 +281,13 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1297/B4, B5 on the bus: the controller sent F to g14 while the canary ace does not run F healthy yet. Started whole,
|
||||
// the assigner wants F on the canary, writes the difference on g14 explained by rule 3, and on ace one unexplained (the
|
||||
// controller sends ace the older fingerprint while ace wants F).
|
||||
func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||
// 1297/B4 on the bus, 1906/D5: the controller sends ace, g14 and shanks builds while the canary ace does not run F healthy yet.
|
||||
// Started whole, the assigner wants F on the canary and writes no difference at all: there is no send to compare with (ADR 0539
|
||||
// §8). 1945/AC19, AC21: shanks, which runs the older fingerprint (composed and granted) with no want, gets its first want at that
|
||||
// fingerprint, never at F; g14, which runs a build of the old form, gets none, so web is on neither list there and the assigner
|
||||
// raises the condition naming g14, web and the canary rule; once ace runs F healthy, g14's first want names F and the condition is
|
||||
// cleared.
|
||||
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
@@ -255,19 +299,21 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||
}
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
for _, fp := range []string{fNew, fOld} {
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
}
|
||||
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
||||
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
|
||||
}
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
|
||||
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-8"), []byte(`{"id":"b-8","repository":"novox/web","source-fingerprint":"`+fOld+`"}`))
|
||||
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-9"), []byte(`{"id":"b-9","repository":"novox/web","source-fingerprint":"`+fNew+`"}`))
|
||||
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
b.start(t)
|
||||
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
@@ -275,18 +321,41 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||
if err != nil || want.FP != fNew {
|
||||
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||
}
|
||||
dkey, _ := delivery.DifferenceKey("g14", "web")
|
||||
d, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, dkey))
|
||||
if err != nil || d.Rule != delivery.RuleCanary || d.Sent != fNew {
|
||||
t.Fatalf("g14's difference = %+v, %v; want rule 3", d, err)
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
|
||||
}
|
||||
akey, _ := delivery.DifferenceKey("ace", "web")
|
||||
a, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, akey))
|
||||
if err != nil || !a.Unexplained || a.Want != fNew || a.Sent != fOld {
|
||||
t.Fatalf("ace's difference = %+v, %v; want it unexplained", a, err)
|
||||
var neither asks.Condition
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
|
||||
neither = c
|
||||
}
|
||||
}
|
||||
}
|
||||
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
|
||||
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
|
||||
}
|
||||
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
|
||||
t.Errorf("differences were written: %v", keys)
|
||||
}
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
|
||||
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
|
||||
t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
|
||||
}
|
||||
|
||||
// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
|
||||
// there by itself: the condition is cleared.
|
||||
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F with its composition", w)
|
||||
}
|
||||
cleared := false
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
|
||||
c, ok := b.conditionsSaid(t)[neither.Key]
|
||||
cleared = ok && c.State == asks.StateCleared
|
||||
}
|
||||
if !cleared {
|
||||
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,12 +375,522 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
|
||||
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
|
||||
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
|
||||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
|
||||
t.Fatalf("held = %+v, %v", h, err)
|
||||
}
|
||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
|
||||
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key.
|
||||
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
|
||||
t.Helper()
|
||||
stream, err := b.js.Stream(b.ctx, "CONDITIONS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]asks.Condition{}
|
||||
cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
batch, err := cons.FetchNoWait(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
c, err := asks.DecodeCondition(m.Data())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[c.Key] = c
|
||||
}
|
||||
if n == 0 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
|
||||
// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node
|
||||
// and the check's words.
|
||||
func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, "mail", mod)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.mail", failed)
|
||||
wantKey, _ := delivery.WantKey("ace", "mail")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
|
||||
b.start(t)
|
||||
|
||||
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
|
||||
t.Fatalf("the restore ask = %+v, %v", a, err)
|
||||
}
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew {
|
||||
t.Errorf("the data module was set back: want = %+v, %v", w, err)
|
||||
}
|
||||
key := ""
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
|
||||
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
|
||||
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
|
||||
}
|
||||
key = k
|
||||
}
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
t.Fatal("no condition for the held fingerprint was said")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants
|
||||
// of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost
|
||||
// and nothing written again.
|
||||
func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
const modules = 40
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
names := make([]string, modules)
|
||||
for i := range names {
|
||||
name := fmt.Sprintf("mod%02d", i)
|
||||
names[i] = name
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
|
||||
digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name))
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, name, mod)
|
||||
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
|
||||
}
|
||||
count := func() int {
|
||||
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return len(keys)
|
||||
}
|
||||
_, stop := b.startStoppable(t)
|
||||
for deadline := time.Now().Add(20 * time.Second); count() == 0 && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
||||
}
|
||||
stop() // replaced, wherever its pass had got to
|
||||
b.start(t)
|
||||
for deadline := time.Now().Add(30 * time.Second); count() < modules && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
}
|
||||
time.Sleep(time.Second) // and no pass of the second writes any again
|
||||
for _, name := range names {
|
||||
key, _ := delivery.WantKey("ace", name)
|
||||
history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Errorf("the want of %s was never written: %v", name, err)
|
||||
continue
|
||||
}
|
||||
if len(history) != 1 {
|
||||
t.Errorf("the want of %s was written %d times, want once", name, len(history))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
|
||||
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
|
||||
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
|
||||
func TestAWantWaitsForItsComposition(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
time.Sleep(time.Second)
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
t.Fatalf("a want was written before the composition existed: %s", e.Value())
|
||||
}
|
||||
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
|
||||
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
|
||||
}
|
||||
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
|
||||
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
|
||||
}
|
||||
// the node's own want
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
|
||||
}
|
||||
ownKey, _ := delivery.ComposedNodeKey("ace")
|
||||
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
|
||||
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the node's own want = %+v, %v", w, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
|
||||
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
t.Helper()
|
||||
digest := b.manifest(t, name, manifest)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, name, mod)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
|
||||
}
|
||||
|
||||
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
|
||||
t.Helper()
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
v, err := delivery.EncodeWant(w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.AssignerWantsBucket, key, v)
|
||||
}
|
||||
|
||||
// wantNow waits until the want of module on node satisfies ok and answers it.
|
||||
func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.Want) bool) delivery.Want {
|
||||
t.Helper()
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
var last delivery.Want
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil {
|
||||
if w, err := delivery.DecodeWant(e.Value()); err == nil {
|
||||
last = w
|
||||
if ok(w) {
|
||||
return w
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("never saw: %s; the want is %+v", what, last)
|
||||
return last
|
||||
}
|
||||
|
||||
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
|
||||
func (b *bed) answer(t *testing.T, key, option string) {
|
||||
t.Helper()
|
||||
a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.answerTo(t, key, option, a.Ask.ID)
|
||||
}
|
||||
|
||||
// answerTo is answer to the ask with this id (the router's warrant names it).
|
||||
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: askID + "@2026-10-24T12:00:00Z", Words: "the operator"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.AnswersBucket, key, v)
|
||||
}
|
||||
|
||||
func (b *bed) hasKey(bucket, key string) bool {
|
||||
_, err := b.buckets[bucket].Get(b.ctx, key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules,
|
||||
// and clears the condition.
|
||||
func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
b.start(t)
|
||||
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld })
|
||||
if w.Newest != fNew {
|
||||
t.Fatalf("the set-back want = %+v", w)
|
||||
}
|
||||
hold, _ := delivery.HoldAskKey("web", fNew)
|
||||
b.get(t, delivery.AssignerAsksBucket, hold)
|
||||
b.answer(t, hold, "release")
|
||||
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
|
||||
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
cleared := false
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
|
||||
}
|
||||
if cleared {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("the condition of the held fingerprint was not cleared")
|
||||
}
|
||||
|
||||
// askNow waits for the backup ask at key to be the one numbered asking and answers it.
|
||||
func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil {
|
||||
if a, err := delivery.DecodeAssignerAsk(key, e.Value()); err == nil && a.Asking == asking {
|
||||
return a
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the ask at %s never became asking %d", key, asking)
|
||||
return delivery.AssignerAsk{}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
|
||||
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
|
||||
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
|
||||
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
|
||||
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
|
||||
b.start(t)
|
||||
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
|
||||
a := b.askNow(t, akey, 0)
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
|
||||
t.Fatalf("the backup ask = %+v", a)
|
||||
}
|
||||
// an answer to another ask is no answer to this one
|
||||
b.answerTo(t, akey, delivery.BackupSkip, "someone-else")
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
|
||||
t.Fatalf("an answer to another ask was taken: %+v", w)
|
||||
}
|
||||
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
|
||||
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
|
||||
if w1.FP != fNew || w1.Composed != first {
|
||||
t.Errorf("the want = %+v", w1)
|
||||
}
|
||||
// the node retries and the backup fails again: a new have value, asked again under the next number
|
||||
again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", again)
|
||||
a2 := b.askNow(t, akey, 2)
|
||||
if a2.Ask.ID == a.Ask.ID {
|
||||
t.Fatalf("asked again under the same id")
|
||||
}
|
||||
b.answerTo(t, akey, delivery.BackupAgain, a2.Ask.ID)
|
||||
w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain && w.Words != w1.Words })
|
||||
if w2.Words == w1.Words {
|
||||
t.Errorf("the second answer left the want as it was: %+v", w2)
|
||||
}
|
||||
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
|
||||
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
|
||||
a3 := b.askNow(t, akey, 3)
|
||||
if !strings.Contains(a3.Ask.Explanation, second) {
|
||||
t.Errorf("the ask for the new revision = %+v", a3)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
|
||||
t.Errorf("the old answer was taken for the new revision: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
|
||||
|
||||
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
|
||||
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
|
||||
// neither is written at once.
|
||||
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, recorded)
|
||||
old := "sha256:" + strings.Repeat("a", 64)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
|
||||
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
|
||||
b.start(t)
|
||||
akey, _ := delivery.RecordAskKey("ace", "web", nw)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
|
||||
t.Fatalf("the record ask = %+v, %v", a, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the move was written before the yes: %+v", w)
|
||||
}
|
||||
b.answer(t, akey, assign.OptionYes)
|
||||
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
|
||||
// a new revision at the same fingerprint, judged to move data, waits as well
|
||||
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
|
||||
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
|
||||
b.get(t, delivery.AssignerAsksBucket, rkey)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
|
||||
t.Fatalf("the revision was written before the yes: %+v", w)
|
||||
}
|
||||
// and a revision judged neither is written at once
|
||||
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
|
||||
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
|
||||
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
|
||||
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
|
||||
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
|
||||
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
|
||||
b.start(t)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
|
||||
}
|
||||
// the controller composes the bus at the fingerprint it runs with a new user list
|
||||
key, _ := delivery.ComposedKey("ace", "bus", fOld)
|
||||
free := "sha256:" + strings.Repeat("e", 64)
|
||||
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
|
||||
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
if w.FP != fOld {
|
||||
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
|
||||
}
|
||||
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
|
||||
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
|
||||
t.Errorf("the user-list revision was asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
|
||||
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
|
||||
// cleared saying why, and the want stays at the fingerprint.
|
||||
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
b.start(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||
b.get(t, delivery.AssignerWantsBucket, hkey)
|
||||
b.get(t, delivery.AssignerAsksBucket, akey)
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
|
||||
continue
|
||||
}
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
|
||||
}
|
||||
|
||||
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
|
||||
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
|
||||
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
_, stop := b.startStoppable(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||
first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the hold was not lifted")
|
||||
}
|
||||
}
|
||||
stop()
|
||||
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
|
||||
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.start(t)
|
||||
again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Ask.ID == first.Ask.ID {
|
||||
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
|
||||
}
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -42,6 +43,8 @@ var readBuckets = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
@@ -130,7 +133,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, logger)
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
|
||||
}
|
||||
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
@@ -138,6 +141,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
var watched = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
@@ -151,7 +156,7 @@ var watched = []string{
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -215,7 +220,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *lo
|
||||
case <-settled:
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
wake()
|
||||
for {
|
||||
@@ -230,3 +235,20 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *lo
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||
// (ADR 0437 clause 9).
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject(delivery.AssignerModule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := asks.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, data)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||
@@ -19,76 +21,55 @@ type passer struct {
|
||||
w *world
|
||||
logf func(string, ...any)
|
||||
sdk string
|
||||
asks *composeAsks
|
||||
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(asks.Condition) error
|
||||
said map[string]bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
asks *composeAsks
|
||||
// now is the clock; nil is time.Now.
|
||||
now func() time.Time
|
||||
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||
// first pass after a start, which clears what a previous process may have raised (startClears).
|
||||
neither map[string]neitherOne
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
if p.now != nil {
|
||||
return p.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.compositions(ctx)
|
||||
rules := p.wants(ctx)
|
||||
p.differences(ctx, rules)
|
||||
p.wants(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
}
|
||||
|
||||
// differences writes, for each node and module the assigner wants or the controller sends, the difference between the
|
||||
// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that
|
||||
// ended; its history keeps what it was.
|
||||
func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) {
|
||||
_, assignments := p.w.assigned()
|
||||
wanted := p.w.wantsOf()
|
||||
trunk := p.w.trunk()
|
||||
pairs := map[[2]string]bool{}
|
||||
for _, a := range assignments {
|
||||
pairs[[2]string{a.Node, a.Module}] = true
|
||||
}
|
||||
for module, byNode := range wanted {
|
||||
for node := range byNode {
|
||||
pairs[[2]string{node, module}] = true
|
||||
}
|
||||
}
|
||||
written := p.w.read(delivery.AssignerDifferencesBucket)
|
||||
keep := map[string]bool{}
|
||||
for pair := range pairs {
|
||||
node, module := pair[0], pair[1]
|
||||
key, err := delivery.DifferenceKey(node, module)
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var want *delivery.Want
|
||||
if w, ok := wanted[module][node]; ok {
|
||||
want = &w
|
||||
}
|
||||
var sent assign.Sent
|
||||
if a, ok := assignments[node+"."+module]; ok {
|
||||
sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)}
|
||||
}
|
||||
d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node])
|
||||
if !ok {
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
continue
|
||||
}
|
||||
keep[key] = true
|
||||
if before, err := delivery.DecodeDifference(written[key]); err == nil {
|
||||
before.V = 0
|
||||
if delivery.SameDifference(before, d) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) })
|
||||
}
|
||||
for key := range written {
|
||||
if keep[key] {
|
||||
continue
|
||||
}
|
||||
if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil {
|
||||
p.logf("the difference %s that ended could not be deleted: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.w.put(delivery.AssignerDifferencesBucket, key, nil)
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
}
|
||||
|
||||
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
|
||||
// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node
|
||||
// back, for the comparison.
|
||||
func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding {
|
||||
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
@@ -109,7 +90,6 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
rules := map[string]map[string]assign.Holding{}
|
||||
names := map[string]bool{}
|
||||
for name := range trunk {
|
||||
names[name] = true
|
||||
@@ -117,20 +97,59 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
for name := range wanted {
|
||||
names[name] = true
|
||||
}
|
||||
neither := map[string]neitherOne{}
|
||||
// The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay
|
||||
// raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every
|
||||
// assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
defer func() { p.sayNeither(neither, stale) }()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]}
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(),
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
return err == nil && p.answeredWith(key, assign.OptionRestore)
|
||||
},
|
||||
Allowed: func(node, digest string) bool {
|
||||
key, err := delivery.RecordAskKey(node, name, digest)
|
||||
return err == nil && p.answeredWith(key, assign.OptionYes)
|
||||
}}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
}
|
||||
if m.Released {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
}
|
||||
last := map[string]string{}
|
||||
for _, node := range m.Nodes {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
rules[name] = out.Rules
|
||||
gatherNeither(neither, name, out)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
}
|
||||
if out.Held != nil {
|
||||
key, err := delivery.HeldKey(name, fp)
|
||||
if err == nil {
|
||||
@@ -138,8 +157,18 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
||||
}
|
||||
}
|
||||
if out.Held != nil || m.Held {
|
||||
p.holdAsk(ctx, name, fp)
|
||||
if out.Lifted {
|
||||
p.lift(ctx, name, fp)
|
||||
}
|
||||
if out.Held != nil || (m.Held && !out.Lifted) {
|
||||
p.holdAsk(ctx, name, fp, out.Held)
|
||||
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
||||
}
|
||||
for _, node := range out.Restore {
|
||||
p.restoreAsk(ctx, name, node, fp)
|
||||
}
|
||||
for _, need := range out.Record {
|
||||
p.recordAsk(ctx, name, fp, need)
|
||||
}
|
||||
for node, want := range out.Wants {
|
||||
key, err := delivery.WantKey(node, name)
|
||||
@@ -151,7 +180,74 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
return rules
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
holding assign.Holding
|
||||
}
|
||||
|
||||
// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module.
|
||||
func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) {
|
||||
for node, h := range out.Neither {
|
||||
into[node+"."+module] = neitherOne{node: node, module: module, holding: h}
|
||||
}
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router.
|
||||
func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; !on {
|
||||
into[node+"."+m.Name] = neitherOne{node: node, module: m.Name}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears,
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
if _, on := now[key]; !on {
|
||||
p.neither[key] = one
|
||||
}
|
||||
}
|
||||
}
|
||||
for key, one := range now {
|
||||
if _, said := p.neither[key]; said {
|
||||
continue
|
||||
}
|
||||
c := assign.NeitherCondition(one.node, one.module, one.holding, true)
|
||||
if p.say != nil {
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
|
||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||
@@ -222,11 +318,219 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
||||
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
||||
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
||||
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
||||
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
|
||||
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
|
||||
key, err := delivery.HoldAskKey(module, fp)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
a := assign.HoldAsk(key, module, fp, time.Now())
|
||||
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
|
||||
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
|
||||
now := p.clock()
|
||||
asking := int(now.Unix())
|
||||
h := p.heldHold(module, fp, fresh)
|
||||
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
|
||||
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
|
||||
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
|
||||
func (p *passer) lift(ctx context.Context, module, fp string) {
|
||||
held, err := delivery.HeldKey(module, fp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ask, err := delivery.HoldAskKey(module, fp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c := assign.LiftedCondition(module, fp)
|
||||
if !p.said[c.Key+".lifted"] {
|
||||
if p.say != nil {
|
||||
if err := p.say(c); err != nil {
|
||||
// The hold stays written, so the next pass lifts it again and says so then.
|
||||
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.logf("%s", assign.LiftedWords(module, fp))
|
||||
p.said[c.Key+".lifted"] = true
|
||||
delete(p.said, c.Key+".raised") // a hold written again is said again
|
||||
}
|
||||
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
||||
}
|
||||
|
||||
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
||||
// pass.
|
||||
func (p *passer) remove(ctx context.Context, bucket, key string) {
|
||||
if !p.w.has(bucket, key) {
|
||||
return
|
||||
}
|
||||
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("%s %s could not be taken away: %v", bucket, key, err)
|
||||
return
|
||||
}
|
||||
p.w.put(bucket, key, nil)
|
||||
}
|
||||
|
||||
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
||||
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
||||
if fresh != nil {
|
||||
return *fresh
|
||||
}
|
||||
key, err := delivery.HeldKey(module, fp)
|
||||
if err != nil {
|
||||
return delivery.Held{}
|
||||
}
|
||||
h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key])
|
||||
return h
|
||||
}
|
||||
|
||||
// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the
|
||||
// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask.
|
||||
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *delivery.Held, data bool) {
|
||||
h := p.heldHold(module, fp, fresh)
|
||||
c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data)
|
||||
if p.say == nil || p.said[c.Key+".raised"] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
|
||||
delete(p.said, c.Key+".cleared") // and released again is cleared again
|
||||
}
|
||||
|
||||
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
||||
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
|
||||
key, err := delivery.RestoreAskKey(node, module, fp)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
words := p.heldHold(module, fp, nil).Words
|
||||
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
|
||||
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
|
||||
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
|
||||
// the same fingerprint, does not carry over to a later one (issue 1906).
|
||||
func (p *passer) answeredWith(key, option string) bool {
|
||||
a, ok := p.w.answer(key)
|
||||
if !ok || a.Option != option {
|
||||
return false
|
||||
}
|
||||
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
id, _, _ := strings.Cut(a.Warrant, "@")
|
||||
return id == ask.Ask.ID
|
||||
}
|
||||
|
||||
// clearHeld clears, once, the condition of a hold the operator released.
|
||||
func (p *passer) clearHeld(module, fp string) {
|
||||
c := assign.HeldCondition(module, fp, "", "", false, false)
|
||||
if p.say == nil || p.said[c.Key+".cleared"] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key+".cleared"] = true
|
||||
}
|
||||
|
||||
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
|
||||
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
|
||||
key, err := delivery.RecordAskKey(need.Node, module, need.Digest)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
|
||||
// it asks whether to back up again or go on without a backup, raises the condition naming node, module and the backup's words,
|
||||
// and writes the answer into the node's want for the module, which the node agent reads from it and from nothing else.
|
||||
//
|
||||
// The ask is bound to the composition (it names the digest, and a new revision at the same fingerprint is asked again under a
|
||||
// new number) and an answer counts only if the router's warrant names the id of the ask now at the key (the warrant's first part,
|
||||
// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again
|
||||
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
|
||||
// in a new have value after the answer was written.
|
||||
func (p *passer) backups(ctx context.Context) {
|
||||
wanted := p.w.wantsOf()
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
have, err := delivery.DecodeHave(value)
|
||||
if err != nil || have.Phase != delivery.PhaseHeld {
|
||||
continue
|
||||
}
|
||||
want, ok := wanted[module][node]
|
||||
if !ok || want.Removed {
|
||||
continue
|
||||
}
|
||||
akey, err := delivery.BackupAskKey(node, module, want.FP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
current, there := delivery.DecodeAssignerAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
|
||||
if there != nil || assign.BackupAskDigest(current) != want.Composed {
|
||||
after := -1
|
||||
if there == nil {
|
||||
after = current.Asking
|
||||
}
|
||||
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(after))
|
||||
continue
|
||||
}
|
||||
answer, answered := p.w.answer(akey)
|
||||
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != current.Ask.ID ||
|
||||
(answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) {
|
||||
continue // not answered yet: the ask stands
|
||||
}
|
||||
words := assign.BackupAnswerWords(answer.Option, current.Asking)
|
||||
haveChanges := p.w.changed(delivery.HaveBucket, key)
|
||||
if want.Backup != answer.Option || want.Words != words {
|
||||
want.Backup, want.Words = answer.Option, words
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
w := want
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
|
||||
p.answeredAt[current.Ask.ID] = haveChanges
|
||||
}
|
||||
continue
|
||||
}
|
||||
if answer.Option != delivery.BackupAgain {
|
||||
continue
|
||||
}
|
||||
// The answer is in the want. The node retried and the backup failed again when the have took another value since.
|
||||
if seen, known := p.answeredAt[current.Ask.ID]; !known {
|
||||
p.answeredAt[current.Ask.ID] = haveChanges // after a restart: wait for the next failure
|
||||
} else if haveChanges > seen {
|
||||
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(current.Asking))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
|
||||
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
|
||||
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
c := assign.BackupCondition(node, module, want.FP, words, true)
|
||||
if p.say == nil || p.said[c.Key] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the failed backup of %s on %s could not be said: %v", module, node, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// memKV is the part of a bucket a pass writes: puts and deletes.
|
||||
type memKV struct {
|
||||
jetstream.KeyValue
|
||||
values map[string][]byte
|
||||
}
|
||||
|
||||
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.values[key] = value
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
|
||||
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
delete(m.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
|
||||
func newPasser(kvs buckets, at time.Time) *passer {
|
||||
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
|
||||
// an id it keeps).
|
||||
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
held := &delivery.Held{Node: "ace", Words: "health failed"}
|
||||
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
|
||||
first.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.lift(context.Background(), "web", fpTest)
|
||||
if _, there := asksKV.values[key]; there {
|
||||
t.Fatal("the lift left the hold ask")
|
||||
}
|
||||
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
|
||||
again.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a2.Ask.ID == a1.Ask.ID {
|
||||
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
||||
// taken away, is not taken for the hold that stands now.
|
||||
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
put := func(asking int) string {
|
||||
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
return a.Ask.ID
|
||||
}
|
||||
first := put(100)
|
||||
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
|
||||
p.w.put(delivery.AnswersBucket, key, v)
|
||||
if !p.answeredWith(key, "release") {
|
||||
t.Fatal("the answer to the ask that stands is not taken")
|
||||
}
|
||||
if p.answeredWith(key, "keep") {
|
||||
t.Error("an answer was taken for an option not chosen")
|
||||
}
|
||||
put(200) // the same fingerprint held again, after a lift
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("the answer to an earlier hold carried over to a later one")
|
||||
}
|
||||
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("an answer to an ask taken away was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
p.say = func(c asks.Condition) error {
|
||||
if failing {
|
||||
failing = false
|
||||
return errors.New("the channel is away")
|
||||
}
|
||||
said = append(said, c)
|
||||
return nil
|
||||
}
|
||||
var logged []string
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil) // the say fails
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
p.w.put(delivery.HaveBucket, "g14.web", have)
|
||||
key, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
p.w.put(delivery.AssignerWantsBucket, key, want)
|
||||
p.wants(context.Background())
|
||||
cleared := map[string]bool{}
|
||||
for _, c := range said {
|
||||
if c.State != asks.StateCleared {
|
||||
t.Errorf("the first pass raised %+v", c)
|
||||
}
|
||||
cleared[c.Key] = true
|
||||
}
|
||||
neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key
|
||||
held := assign.HeldCondition("web", fpTest, "", "", false, false).Key
|
||||
if !cleared[neither] || !cleared[held] {
|
||||
t.Errorf("cleared = %v, want %s and %s", cleared, neither, held)
|
||||
}
|
||||
n := len(said)
|
||||
p.wants(context.Background())
|
||||
if len(said) != n {
|
||||
t.Errorf("a second pass said again: %+v", said[n:])
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]asks.Condition{}
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
|
||||
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
|
||||
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
|
||||
p.w.put(delivery.AssignerWantsBucket, hkey, held)
|
||||
return p, said
|
||||
}
|
||||
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
|
||||
neithers := func(said []asks.Condition) (raised, cleared int) {
|
||||
for _, c := range said {
|
||||
if c.Key != key {
|
||||
continue
|
||||
}
|
||||
if c.State == asks.StateRaised {
|
||||
raised++
|
||||
} else {
|
||||
cleared++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
// a standing neither this process raised
|
||||
p, said := setup()
|
||||
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: delivery.RuleNotHeld}}}
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||
t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||
}
|
||||
// one a previous process raised: the first pass after a start clears it
|
||||
p, said = setup()
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||
}
|
||||
}
|
||||
@@ -44,8 +44,8 @@ type world struct {
|
||||
manifests map[string]parsed
|
||||
// healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to.
|
||||
healthy map[string][]string
|
||||
// sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one.
|
||||
sent map[string]string
|
||||
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
|
||||
changes map[string]uint64
|
||||
}
|
||||
|
||||
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||
@@ -56,7 +56,7 @@ type parsed struct {
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}}
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
|
||||
}
|
||||
|
||||
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||
@@ -71,6 +71,7 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
return
|
||||
}
|
||||
w.values[bucket][key] = value
|
||||
w.changes[bucket+"|"+key]++
|
||||
if bucket == delivery.HaveBucket {
|
||||
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||
@@ -325,6 +326,13 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
return n
|
||||
}
|
||||
|
||||
// value is what a bucket holds at a key now, nil when nothing.
|
||||
func (w *world) value(bucket, key string) []byte {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
w.mu.Lock()
|
||||
@@ -349,30 +357,55 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record
|
||||
// (mesh-controller_builds `<module>.build.<id>`) once and kept; empty when the record names none.
|
||||
func (w *world) sentFP(ctx context.Context, get getter, module, build string) string {
|
||||
if build == "" {
|
||||
return ""
|
||||
}
|
||||
w.mu.Lock()
|
||||
fp, ok := w.sent[build]
|
||||
w.mu.Unlock()
|
||||
if ok {
|
||||
return fp
|
||||
}
|
||||
value, err := get(ctx, delivery.BuildKey(module, build))
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var rec delivery.BuildRecord
|
||||
if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) {
|
||||
return ""
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.sent[build] = rec.SourceFingerprint
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
return rec.SourceFingerprint
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
func (w *world) answer(key string) (delivery.Answer, bool) {
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.AnswersBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.Answer{}, false
|
||||
}
|
||||
a, err := delivery.DecodeAnswer(value)
|
||||
return a, err == nil
|
||||
}
|
||||
|
||||
// changed is how many values the key has taken in this process (see world.changes).
|
||||
func (w *world) changed(bucket, key string) uint64 {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.changes[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
|
||||
|
||||
@@ -111,3 +111,17 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
|
||||
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
|
||||
t.Errorf("changed = %d, want 2", got)
|
||||
}
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 h1:ZEEeYibEPD1rngOAqbODzRY5rUStGC46OpyQwtNCYbs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -2,24 +2,28 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
||||
// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer
|
||||
// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6.
|
||||
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
|
||||
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
|
||||
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
||||
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
||||
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
||||
"no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
||||
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
||||
"lets the wants move to it now.", module, node, short(fp), words, module),
|
||||
Options: []asks.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
@@ -27,3 +31,176 @@ func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
||||
const (
|
||||
OptionRestore = "restore"
|
||||
OptionLeaveStopped = "leave-stopped"
|
||||
)
|
||||
|
||||
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
|
||||
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
|
||||
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
|
||||
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
|
||||
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
|
||||
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module stays as it is on the node",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
|
||||
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
|
||||
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
|
||||
// data, whose restore ask waits for the operator.
|
||||
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
|
||||
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
|
||||
if needsYou {
|
||||
c.Needs = "answer whether to restore " + module + " on " + node
|
||||
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
|
||||
} else {
|
||||
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
// node and module, and cleared (raised false) when the node has a want for it again.
|
||||
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".neither." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is on no list", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
c.Resolved = LiftedWords(module, fp) + "."
|
||||
return c
|
||||
}
|
||||
|
||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||
const (
|
||||
OptionYes = "yes"
|
||||
OptionNo = "no"
|
||||
OptionBackupAgain = delivery.BackupAgain
|
||||
OptionBackupSkip = delivery.BackupSkip
|
||||
)
|
||||
|
||||
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
|
||||
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
|
||||
// after the answer yes. The way the move was judged is in the ask.
|
||||
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + need.Words,
|
||||
}[need.Judgement]
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
|
||||
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
|
||||
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
|
||||
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
|
||||
func NextBackupAsking(after int) int {
|
||||
if after < 0 {
|
||||
return 0
|
||||
}
|
||||
return max(after+1, 2)
|
||||
}
|
||||
|
||||
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
|
||||
|
||||
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
|
||||
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
|
||||
// under a new number (ADR 0539 §16).
|
||||
func BackupAskDigest(a delivery.AssignerAsk) string {
|
||||
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
|
||||
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
|
||||
func BackupAnswerWords(option string, asking int) string {
|
||||
return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
|
||||
}
|
||||
|
||||
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
|
||||
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
|
||||
// backup's own words, from the node's have.
|
||||
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
|
||||
module, short(fp), node, words, module, digest),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module keeps running what it has",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
|
||||
// naming the node, the module and the backup's words; cleared under the same key.
|
||||
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
|
||||
needs := "answer whether to back up again or go on without a backup"
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -15,7 +17,39 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := HoldAsk(key, "mail", fNew, now)
|
||||
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
||||
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
||||
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
||||
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
|
||||
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
|
||||
t.Errorf("the ask asked again cannot be written: %v", err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
|
||||
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
|
||||
func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
@@ -25,4 +59,107 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
var ids []string
|
||||
for _, o := range a.Ask.Options {
|
||||
ids = append(ids, o.ID)
|
||||
}
|
||||
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
|
||||
t.Errorf("options = %v", ids)
|
||||
}
|
||||
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
|
||||
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
now := time.Now()
|
||||
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
|
||||
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
|
||||
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("%s: the router refuses the ask: %v", key, err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("%s: cannot be written: %v", key, err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("%s: the view may not answer it: %v", key, err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
|
||||
t.Errorf("the record ask = %+v", r.Ask)
|
||||
}
|
||||
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", b.Ask)
|
||||
}
|
||||
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
|
||||
t.Errorf("asked again under the same id")
|
||||
}
|
||||
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
|
||||
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
|
||||
}
|
||||
if BackupAskDigest(b) != dgst {
|
||||
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
|
||||
}
|
||||
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
|
||||
t.Errorf("two answers of the same option are one want value")
|
||||
}
|
||||
c := BackupCondition("ace", "mail", fNew, "no space left", true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||
// under the assigner's own name keyed by node and module, and is cleared under the same key.
|
||||
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
|
||||
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its
|
||||
// assignment names, and that build's fingerprint of the new form (empty when its record names none).
|
||||
type Sent struct {
|
||||
Sends bool
|
||||
Build string
|
||||
FP string
|
||||
}
|
||||
|
||||
// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4),
|
||||
// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and
|
||||
// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the
|
||||
// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said
|
||||
// in words, for acceptance test 17 of design 47 to read.
|
||||
func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) {
|
||||
var d delivery.Difference
|
||||
removed := want != nil && want.Removed
|
||||
if want != nil && !removed {
|
||||
d.Want = want.FP
|
||||
}
|
||||
d.WantRemoved = removed
|
||||
if !sent.Sends {
|
||||
if want == nil || removed {
|
||||
return delivery.Difference{}, false
|
||||
}
|
||||
d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing"
|
||||
return d, true
|
||||
}
|
||||
d.Sent, d.SentBuild = sent.FP, sent.Build
|
||||
switch {
|
||||
case sent.FP == "":
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build)
|
||||
case removed:
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP))
|
||||
case sent.FP == d.Want:
|
||||
return delivery.Difference{}, false
|
||||
case hold.Rule != 0 && sent.FP == newest:
|
||||
d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words)
|
||||
case d.Want == "":
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP))
|
||||
default:
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want))
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3.
|
||||
func TestADifferenceExplainedByTheCanary(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary}
|
||||
d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"])
|
||||
if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew {
|
||||
t.Fatalf("difference = %+v, %v; want rule 3", d, ok)
|
||||
}
|
||||
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||
t.Errorf("the difference cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written
|
||||
// unexplained, with words, and is writable.
|
||||
func TestCompare(t *testing.T) {
|
||||
w := delivery.Want{FP: fNew}
|
||||
if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok {
|
||||
t.Errorf("an equal want and send is a difference")
|
||||
}
|
||||
if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok {
|
||||
t.Errorf("no want and nothing sent is a difference")
|
||||
}
|
||||
removed := delivery.Want{Removed: true}
|
||||
if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok {
|
||||
t.Errorf("a removed want and nothing sent is a difference")
|
||||
}
|
||||
old := delivery.Want{FP: fOld}
|
||||
for name, c := range map[string]struct {
|
||||
want *delivery.Want
|
||||
sent Sent
|
||||
}{
|
||||
"sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}},
|
||||
"names no fingerprint": {&w, Sent{Sends: true, Build: "b"}},
|
||||
"sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
"sends nothing": {&w, Sent{}},
|
||||
"no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
"no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
} {
|
||||
d, ok := Compare(c.want, c.sent, fNew, Holding{})
|
||||
if !ok || !d.Unexplained || d.Words == "" {
|
||||
t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok)
|
||||
continue
|
||||
}
|
||||
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||
t.Errorf("%s: the difference cannot be written: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -13,13 +15,42 @@ import (
|
||||
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
||||
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
||||
type Module struct {
|
||||
Name string
|
||||
FP string
|
||||
Built bool
|
||||
Held bool
|
||||
Granted bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
Name string
|
||||
FP string
|
||||
Built bool
|
||||
Held bool
|
||||
Granted bool
|
||||
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
|
||||
// that runs an older one names it (issue 1945). Nil: none.
|
||||
GrantedAt func(fp string) bool
|
||||
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
|
||||
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
|
||||
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||
GrantedOn func(node string) bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
Allowed func(node, digest string) bool
|
||||
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
||||
Released bool
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
||||
Restored func(node string) bool
|
||||
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
|
||||
Kept bool
|
||||
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if m.Composed == nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return m.Composed(node, fp)
|
||||
}
|
||||
|
||||
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
|
||||
@@ -51,27 +82,43 @@ func Canary(nodes []string, online map[string]bool) string {
|
||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||
// limit (ADR 0528 clause 3).
|
||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
return holds(m, node, n, true)
|
||||
}
|
||||
|
||||
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
|
||||
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
|
||||
// composition.
|
||||
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
switch {
|
||||
case !m.Built:
|
||||
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
|
||||
case m.Held:
|
||||
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
||||
}
|
||||
canary := Canary(m.Nodes, n.Online)
|
||||
switch {
|
||||
case canary == "":
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
if ordering {
|
||||
canary := Canary(m.Nodes, n.Online)
|
||||
switch {
|
||||
case canary == "":
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
}
|
||||
}
|
||||
if !m.Granted {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
}
|
||||
if m.GrantedOn != nil && !m.GrantedOn(node) {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
||||
short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
@@ -117,8 +164,30 @@ type Outcome struct {
|
||||
Wants map[string]delivery.Want
|
||||
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||
Held *delivery.Held
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why, for the comparison.
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
// back: the operator is asked whether to restore each (ADR 0539 §19).
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
Rules map[string]Holding
|
||||
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||
// own composition and is never here.
|
||||
Neither map[string]Holding
|
||||
}
|
||||
|
||||
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
||||
// controller's judgement of the move and its words.
|
||||
type RecordNeed struct {
|
||||
Node string
|
||||
Digest string
|
||||
Judgement delivery.Judgement
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why.
|
||||
@@ -130,15 +199,18 @@ type Holding struct {
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||
// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A
|
||||
// module holding data is not set back but asked about (Restore). A
|
||||
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
|
||||
held := m.Held
|
||||
if !held && m.FP != "" {
|
||||
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
|
||||
out.Lifted, held = true, false
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
||||
out.Held = &delivery.Held{Node: node, Words: h.Detail}
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
@@ -152,30 +224,117 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
before, had := wants[node]
|
||||
var next delivery.Want
|
||||
switch ok, rule, words := Holds(mm, node, n); {
|
||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||
if w, ok := m.userListRevision(node, before, had, n); ok {
|
||||
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
continue
|
||||
}
|
||||
ok, rule, words := Holds(mm, node, n)
|
||||
target := m.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
|
||||
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
|
||||
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
|
||||
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
|
||||
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
|
||||
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
|
||||
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
|
||||
// order the move from it as from any want.
|
||||
switch _, composed := m.composition(node, have.FP); {
|
||||
case !ran:
|
||||
case have.FP == m.FP:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
|
||||
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
|
||||
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
|
||||
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
|
||||
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
|
||||
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
|
||||
ok, target = true, have.FP
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case ok:
|
||||
next = delivery.Want{FP: m.FP}
|
||||
d, _ := m.composition(node, target)
|
||||
next = delivery.Want{FP: target, Composed: d.Digest}
|
||||
at := m
|
||||
if target != m.FP {
|
||||
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
if had && !before.Removed && before.FP == target && before.Composed == d.Digest {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
// A removed want is no want: a first want after it goes through the record ask as a first want does.
|
||||
asked, hadLive := before, had
|
||||
if before.Removed {
|
||||
asked, hadLive = delivery.Want{}, false
|
||||
}
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
} else if !hadLive && ran && runs(running) {
|
||||
// The bus, which the assigner never moves (§15), runs here with no want: it is on neither list until the operator
|
||||
// writes its want, and the pass says why (issue 1945).
|
||||
out.Neither[node] = Holding{Rule: delivery.RuleComposed, Words: busMoveWords(m.Name, node, target, d)}
|
||||
}
|
||||
continue
|
||||
}
|
||||
case held && had && before.FP == m.FP:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
last := lastHealthy[node]
|
||||
switch {
|
||||
case keepsData(m.Manifest):
|
||||
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP))
|
||||
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
|
||||
out.Restore = append(out.Restore, node)
|
||||
case last == "":
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
|
||||
default:
|
||||
next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words}
|
||||
d, there := m.composition(node, last)
|
||||
if !there {
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
|
||||
break
|
||||
}
|
||||
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
|
||||
}
|
||||
default:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
if !had || before.Removed {
|
||||
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
|
||||
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||
if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
|
||||
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||
}
|
||||
continue
|
||||
}
|
||||
next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words}
|
||||
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
|
||||
if before.FP == m.FP {
|
||||
next = delivery.Want{FP: m.FP}
|
||||
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
|
||||
next = delivery.Want{FP: m.FP, Composed: before.Composed}
|
||||
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
|
||||
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
|
||||
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
|
||||
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
|
||||
at := m
|
||||
at.FP = before.FP
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
if !wait {
|
||||
next.Composed = d.Digest
|
||||
}
|
||||
}
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||
@@ -190,6 +349,58 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||
// moving). A failed health while it
|
||||
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
|
||||
// the world's lastHealthy) already asks for phase running.
|
||||
func failed(h delivery.Have) bool {
|
||||
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h delivery.Have) bool {
|
||||
return h.Phase == delivery.PhaseRunning
|
||||
}
|
||||
|
||||
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
|
||||
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
|
||||
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
|
||||
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
|
||||
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
|
||||
func allRunHealthy(m Module, n Nodes) bool {
|
||||
ran := 0
|
||||
for _, node := range m.Nodes {
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
|
||||
continue
|
||||
}
|
||||
if !healthy(h, m.FP) {
|
||||
return false
|
||||
}
|
||||
ran++
|
||||
}
|
||||
return ran > 0
|
||||
}
|
||||
|
||||
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
|
||||
// it, and the node's own words. The node is the held record's own field.
|
||||
func HeldWords(h delivery.Have, at time.Time) string {
|
||||
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
|
||||
if !at.IsZero() {
|
||||
words += ", seen at " + at.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if d := strings.TrimSpace(h.Detail); d != "" {
|
||||
return words + ": " + d
|
||||
}
|
||||
return words + ": the node gave no words"
|
||||
}
|
||||
|
||||
// LiftedWords say why a hold was lifted by itself.
|
||||
func LiftedWords(module, fp string) string {
|
||||
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
|
||||
}
|
||||
|
||||
// normal is a want without its version, as a decoded and a composed one are compared.
|
||||
func normal(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
@@ -203,3 +414,117 @@ func short(fp string) string {
|
||||
}
|
||||
return part[:12]
|
||||
}
|
||||
|
||||
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
|
||||
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
|
||||
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
|
||||
// same fingerprint judged neither (a user list only, §7).
|
||||
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
|
||||
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
|
||||
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
|
||||
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
|
||||
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
prev := before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
if m.Allowed != nil && m.Allowed(node, d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// busMoveWords say why the bus module's want at fp is not written on node: its composition differs from what the node runs in more
|
||||
// than the bus's user list, so it is a move of the bus, which the operator writes (ADR 0539 §15).
|
||||
func busMoveWords(module, node, fp string, d delivery.ComposedDigest) string {
|
||||
why := string(d.Judgement)
|
||||
if w := strings.TrimSpace(d.Words); w != "" {
|
||||
why += ": " + w
|
||||
}
|
||||
return fmt.Sprintf("the composition of %s for %s at %s changes more than the bus's user list (judged %s), so it is a move of the bus, "+
|
||||
"which the assigner never makes: the operator writes its want (ADR 0539 §15)", module, node, short(fp), why)
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
if d.Policy != "" {
|
||||
return d.Policy == delivery.PolicyRecord
|
||||
}
|
||||
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
|
||||
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
|
||||
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
|
||||
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
|
||||
//
|
||||
// A node whose have is of the old form (from the old node-engine: a build and no fingerprint of the new form, no composition) runs
|
||||
// no fingerprint a composition is kept at, so its revision names F, the newest fingerprint, when the controller judged F's
|
||||
// composition to differ from the declaration the node last applied only in the bus's user list (ADR 0539 §7, ADR 0540 §9). Any
|
||||
// other difference is a move of the bus and stays the operator's (§15). Without this the bus never got its first want after the
|
||||
// switch, and its user list was never written again (issue 1945: novox's nats from 11:23 on 2026-10-25).
|
||||
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
|
||||
if !catalogue.ProvidesBus(m.Manifest) {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if ran && oldForm(running) && m.FP != "" && (!had || before.Removed || before.FP == m.FP) {
|
||||
d, ok := m.composition(node, m.FP)
|
||||
if !ok || !d.UserListOnly {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
w := delivery.Want{FP: m.FP, Composed: d.Digest}
|
||||
if had && !before.Removed && before.Composed == d.Digest {
|
||||
w.Backup, w.Words = before.Backup, before.Words
|
||||
}
|
||||
return w, true
|
||||
}
|
||||
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
d, ok := m.composition(node, running.FP)
|
||||
if !ok || !d.UserListOnly {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
w := delivery.Want{FP: running.FP, Composed: d.Digest}
|
||||
if had && before.Composed == d.Digest {
|
||||
w.Backup, w.Words = before.Backup, before.Words
|
||||
}
|
||||
return w, true
|
||||
}
|
||||
|
||||
// oldForm says whether a have is of the old form: the old node-engine's, naming a build and no fingerprint of the new form, and no
|
||||
// composition (issue 1945: novox's nats, build-1792674595610176744).
|
||||
func oldForm(h delivery.Have) bool {
|
||||
return h.Phase != delivery.PhaseRemoved && h.Composed == "" && (h.FP != "" || h.Build != "") &&
|
||||
!strings.HasPrefix(h.FP, delivery.SourcePrefix2)
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node: its own `<node>.<module>` or a process's
|
||||
// `<node>.<module>.<process>`. A grant that names no account (a module that does not speak on the bus) names every node.
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
if len(g.Accounts) == 0 {
|
||||
return true
|
||||
}
|
||||
own := node + "." + module
|
||||
for _, a := range g.Accounts {
|
||||
if a == own || strings.HasPrefix(a, own+".") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -18,7 +20,7 @@ var (
|
||||
// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider
|
||||
// store healthy on ace and the canary ace running fNew healthy.
|
||||
func ready() (Module, Nodes) {
|
||||
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Nodes: []string{"g14", "ace"},
|
||||
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"},
|
||||
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
|
||||
n := Nodes{
|
||||
Online: map[string]bool{"ace": true, "g14": true},
|
||||
@@ -31,8 +33,16 @@ func ready() (Module, Nodes) {
|
||||
return m, n
|
||||
}
|
||||
|
||||
// dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated.
|
||||
func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) }
|
||||
|
||||
// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither.
|
||||
func composedEverywhere(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, true
|
||||
}
|
||||
|
||||
func wantedOld() map[string]delivery.Want {
|
||||
return map[string]delivery.Want{"ace": {FP: fOld}, "g14": {FP: fOld}}
|
||||
return map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}, "g14": {FP: fOld, Composed: dig(fOld)}}
|
||||
}
|
||||
|
||||
// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so
|
||||
@@ -41,7 +51,7 @@ func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
|
||||
m, n := ready()
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 {
|
||||
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
|
||||
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
|
||||
}
|
||||
}
|
||||
@@ -59,6 +69,9 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
},
|
||||
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
|
||||
delivery.RuleComposed: func(m *Module, _ *Nodes) {
|
||||
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
|
||||
},
|
||||
} {
|
||||
m, n := ready()
|
||||
breakIt(&m, &n)
|
||||
@@ -73,6 +86,33 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
||||
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
||||
m, n := ready()
|
||||
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew {
|
||||
t.Errorf("ace, whose account the grant names, wants %+v", w)
|
||||
}
|
||||
if got := out.Rules["g14"]; got.Rule != delivery.RuleGrant || !strings.Contains(got.Words, "on g14") {
|
||||
t.Errorf("g14, whose account the grant does not name, is held by %+v", got)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
accounts []string
|
||||
node, module string
|
||||
named bool
|
||||
}{
|
||||
{nil, "g14", "mail", true},
|
||||
{[]string{"g14.mail.send"}, "g14", "mail", true},
|
||||
{[]string{"g14.mailer"}, "g14", "mail", false},
|
||||
{[]string{"ace.mail"}, "g14", "mail", false},
|
||||
} {
|
||||
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
|
||||
@@ -98,11 +138,13 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
|
||||
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}, "g14": {FP: fOld}}, map[string]string{"ace": fOld})
|
||||
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
|
||||
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
||||
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
|
||||
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
|
||||
t.Fatalf("held = %+v", out.Held)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w)
|
||||
}
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
@@ -122,6 +164,20 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
|
||||
if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") {
|
||||
t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w)
|
||||
}
|
||||
// 1906/D3: and the node is to be asked whether to restore it
|
||||
if !slices.Equal(out.Restore, []string{"ace"}) {
|
||||
t.Errorf("restore asks = %v, want ace", out.Restore)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: a module without data is set back and asked about nothing.
|
||||
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
|
||||
if len(out.Restore) != 0 {
|
||||
t.Errorf("restore asks = %v", out.Restore)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that
|
||||
@@ -129,7 +185,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
|
||||
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
|
||||
m, n := ready()
|
||||
m.Nodes = []string{"ace"}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew}, "g14": {FP: fOld}}, nil)
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
||||
if w := out.Wants["g14"]; !w.Removed {
|
||||
t.Errorf("g14's want = %+v, want removed", w)
|
||||
}
|
||||
@@ -137,3 +193,765 @@ func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
|
||||
t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its
|
||||
// digest once it exists; a node with no want before still gets none until then.
|
||||
func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
|
||||
m, n := ready()
|
||||
have := map[string]bool{}
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if !have[node+"|"+fp] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
for node, w := range out.Wants {
|
||||
if w.FP == fNew {
|
||||
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
|
||||
}
|
||||
}
|
||||
if r := out.Rules["ace"]; r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "ace") {
|
||||
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
|
||||
}
|
||||
// and a node that never had a want gets none
|
||||
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
|
||||
t.Errorf("wants without a composition: %+v", out.Wants)
|
||||
}
|
||||
have["ace|"+fNew] = true
|
||||
out = Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
|
||||
}
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
t.Errorf("g14, whose composition does not exist, moved: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want,
|
||||
// by the same rules: written when they hold, and not while one does not.
|
||||
func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
|
||||
m, n := ready()
|
||||
revised := "sha256:" + strings.Repeat("9", 64)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedNeither}, true
|
||||
}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
||||
out := Wants(m, n, running, nil)
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
if w := out.Wants[node]; w.FP != fNew || w.Composed != revised {
|
||||
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
|
||||
}
|
||||
}
|
||||
m.Granted = false
|
||||
out = Wants(m, n, running, nil)
|
||||
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
|
||||
t.Errorf("a revision was written while a rule holds: %+v", w)
|
||||
}
|
||||
|
||||
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
|
||||
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
|
||||
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
|
||||
out = Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["g14"]; w.FP != fOld || w.Composed != revised || w.Newest != fNew {
|
||||
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
|
||||
}
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord,
|
||||
Against: dig(fOld), Words: "recreates"}, true
|
||||
}
|
||||
out = Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["g14"]; w.Composed == revised {
|
||||
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
|
||||
}
|
||||
if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) {
|
||||
t.Errorf("the revision was not asked: %+v", out.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
|
||||
func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if fp == fOld {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
out := Wants(m, n, running, map[string]string{"ace": fOld})
|
||||
if w := out.Wants["ace"]; w.FP == fOld {
|
||||
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
|
||||
}
|
||||
m.Composed = composedEverywhere
|
||||
out = Wants(m, n, running, map[string]string{"ace": fOld})
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
|
||||
t.Errorf("set-back want = %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F).
|
||||
func movingTo(j delivery.Judgement) (Module, Nodes, map[string]delivery.Want) {
|
||||
m, n := ready()
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: j}
|
||||
if j != delivery.JudgedNeither {
|
||||
d.Against, d.Words = dig(fOld), "judged"
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
return m, n, wantedOld()
|
||||
}
|
||||
|
||||
func recording(m *Module) {
|
||||
m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be
|
||||
// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint.
|
||||
func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) {
|
||||
for _, j := range []delivery.Judgement{delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot} {
|
||||
m, n, wanted := movingTo(j)
|
||||
recording(&m)
|
||||
out := Wants(m, n, wanted, nil)
|
||||
if w, ok := out.Wants["ace"]; ok {
|
||||
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
|
||||
}
|
||||
var ace *RecordNeed
|
||||
for i := range out.Record {
|
||||
if out.Record[i].Node == "ace" {
|
||||
ace = &out.Record[i]
|
||||
}
|
||||
}
|
||||
if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j {
|
||||
t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record)
|
||||
}
|
||||
// a new revision at the same fingerprint is asked too
|
||||
same := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}}
|
||||
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
|
||||
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
|
||||
}
|
||||
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
|
||||
out = Wants(m, n, wanted, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("%s: after the yes ace's want = %+v", j, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module
|
||||
// that rolls (not record) is never asked.
|
||||
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
|
||||
m, n, wanted := movingTo(delivery.JudgedNeither)
|
||||
recording(&m)
|
||||
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("neither: %+v", out)
|
||||
}
|
||||
m, n, _ = movingTo(delivery.JudgedRecreates)
|
||||
recording(&m)
|
||||
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
|
||||
delete(n.Haves["ace"], "mail")
|
||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("a node that never ran it: %+v", out)
|
||||
}
|
||||
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
|
||||
m, n, _ = movingTo(delivery.JudgedCannot)
|
||||
recording(&m)
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew || len(out.Record) == 0 {
|
||||
t.Errorf("a node that runs it with no want: %+v", out)
|
||||
}
|
||||
m, n, wanted = movingTo(delivery.JudgedRecreates)
|
||||
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("a rolled module: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
|
||||
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
|
||||
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
|
||||
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
|
||||
}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
|
||||
out := Wants(m, n, running, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
|
||||
}
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
||||
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
||||
m, n := ready()
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew), Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
||||
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer was erased: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
||||
// wants move by the rules again.
|
||||
func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
|
||||
m.Released = true
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleNotHeld}}, map[string]string{"ace": fOld})
|
||||
if out.Held != nil {
|
||||
t.Errorf("a released fingerprint was held again: %+v", out.Held)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) || w.Rule != 0 {
|
||||
t.Errorf("ace's want = %+v, want the held fingerprint written again", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes;
|
||||
// answered "leave stopped" (or not answered) it stays.
|
||||
func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
|
||||
m, n := ready()
|
||||
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
|
||||
m.Restored = func(node string) bool { return false }
|
||||
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.FP == fOld {
|
||||
t.Errorf("set back without the operator's restore: %+v", w)
|
||||
}
|
||||
m.Restored = func(node string) bool { return node == "ace" }
|
||||
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
|
||||
t.Errorf("after restore ace's want = %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included;
|
||||
// a composition with none (made by hand) falls back to what the manifest derives.
|
||||
func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
|
||||
m, n, wanted := movingTo(delivery.JudgedMovesData)
|
||||
with := func(policy, from string) {
|
||||
inner := m.Composed
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
d, ok := inner(node, fp)
|
||||
d.Policy, d.PolicyFrom = policy, from
|
||||
return d, ok
|
||||
}
|
||||
}
|
||||
with(delivery.PolicyRecord, "person") // a roll module the operator set to record
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].FP == fNew {
|
||||
t.Errorf("the operator's record policy was not honoured: %+v", out)
|
||||
}
|
||||
recording(&m) // the manifest says record, the operator chose roll
|
||||
with("roll", "person")
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
|
||||
t.Errorf("the operator's roll policy was not honoured: %+v", out)
|
||||
}
|
||||
with("", "") // made by hand: the manifest decides
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
|
||||
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
|
||||
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
|
||||
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
|
||||
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written
|
||||
// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it
|
||||
// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against
|
||||
// the composition the node runs, policy record, UserListOnly set.
|
||||
func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedRecreates)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
userList := "sha256:" + strings.Repeat("7", 64)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
|
||||
Policy: delivery.PolicyRecord, PolicyFrom: "the bus"}
|
||||
if fp == fOld {
|
||||
d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
for name, wanted := range map[string]map[string]delivery.Want{
|
||||
"a want at the fingerprint it runs": {"ace": {FP: fOld, Composed: dig(fOld)}},
|
||||
"no readable want": nil,
|
||||
} {
|
||||
out := Wants(m, n, wanted, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != userList {
|
||||
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
|
||||
}
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record)
|
||||
}
|
||||
}
|
||||
revision := m.Composed
|
||||
// Issue 1945: a have of the old form (a build, no fingerprint of the new form, no composition) runs no fingerprint a composition is
|
||||
// kept at. The revision then names F when the controller judged F's composition to differ from the declaration the node last
|
||||
// applied only in the user list; any other difference is a move of the bus, refused by §15 and said on neither list.
|
||||
for name, c := range map[string]struct {
|
||||
userListOnly bool
|
||||
wanted map[string]delivery.Want
|
||||
want *delivery.Want // nil: no want written
|
||||
neither bool
|
||||
}{
|
||||
"old form, user list only, no want": {userListOnly: true, want: &delivery.Want{FP: fNew, Composed: userList}},
|
||||
"old form, user list only, a removed want": {userListOnly: true, wanted: map[string]delivery.Want{"ace": {Removed: true}}, want: &delivery.Want{FP: fNew, Composed: userList}},
|
||||
"old form, user list only, written before": {userListOnly: true, wanted: map[string]delivery.Want{"ace": {FP: fNew, Composed: userList}}},
|
||||
"old form, a real move of the bus": {neither: true},
|
||||
} {
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if fp != fNew {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
if c.userListOnly {
|
||||
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true, Against: dig(fOld),
|
||||
Words: "only the bus's user list changes, against the declaration ace last applied", Policy: delivery.PolicyRecord}, true
|
||||
}
|
||||
return delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedNeither, Words: "the node runs no composition of mail",
|
||||
Policy: delivery.PolicyRecord}, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{Build: "build-1792674595610176744", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m, n, c.wanted, nil)
|
||||
w, ok := out.Wants["ace"]
|
||||
switch {
|
||||
case c.want == nil && ok:
|
||||
t.Errorf("%s: a want was written: %+v", name, w)
|
||||
case c.want != nil && (!ok || w.FP != c.want.FP || w.Composed != c.want.Composed || w.Removed):
|
||||
t.Errorf("%s: ace's want = %+v (written %v), want %+v", name, w, ok, *c.want)
|
||||
}
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("%s: asked: %+v", name, out.Record)
|
||||
}
|
||||
h, on := out.Neither["ace"]
|
||||
if on != c.neither || (on && (!strings.Contains(h.Words, "§15") || !strings.Contains(h.Words, "more than the bus's user list"))) {
|
||||
t.Errorf("%s: on neither list %v (%+v), want %v with the words of §15", name, on, h, c.neither)
|
||||
}
|
||||
}
|
||||
// the same flag on a module that is not the bus changes nothing
|
||||
m.Manifest.Provides, m.Composed = nil, revision
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
|
||||
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine
|
||||
// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F
|
||||
// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the
|
||||
// declaration it kept); a new fingerprint still waits for its ask.
|
||||
func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
|
||||
for name, bus := range map[string]bool{"a record module": false, "the bus": true} {
|
||||
// only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold
|
||||
unchanged := func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedUnchanged}, true
|
||||
}
|
||||
m, n, _ := movingTo(delivery.JudgedCannot)
|
||||
recording(&m)
|
||||
m.Nodes = []string{"ace"}
|
||||
if bus {
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
}
|
||||
m.Composed = unchanged
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} // no composition named
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
|
||||
}
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("%s: asked at the switch: %+v", name, out.Record)
|
||||
}
|
||||
// a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge,
|
||||
// still wait for the ask
|
||||
for what, j := range map[string]delivery.Judgement{"differing": delivery.JudgedRecreates, "unjudged": delivery.JudgedCannot, "judged neither": delivery.JudgedNeither} {
|
||||
m2, n2, _ := movingTo(j)
|
||||
recording(&m2)
|
||||
m2.Nodes = []string{"ace"}
|
||||
if bus {
|
||||
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
}
|
||||
n2.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m2, n2, nil, nil)
|
||||
if j == delivery.JudgedNeither {
|
||||
if out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("%s, %s: %+v", name, what, out)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 {
|
||||
t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record)
|
||||
}
|
||||
}
|
||||
// a have that names a composition is judged as before
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew && len(out.Record) == 0 {
|
||||
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
|
||||
}
|
||||
// a new fingerprint still asks (the bus is barred instead)
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["ace"]; ok {
|
||||
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
|
||||
}
|
||||
if !bus && len(out.Record) == 0 {
|
||||
t.Errorf("%s: a new fingerprint was not asked", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
|
||||
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
|
||||
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
|
||||
t.Fatalf("held = %+v", out.Held)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
|
||||
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
|
||||
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
|
||||
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
everywhere := func() (Module, Nodes) {
|
||||
m, n := ready()
|
||||
m.Held = true
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return m, n
|
||||
}
|
||||
m, n := everywhere()
|
||||
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
|
||||
if !out.Lifted || out.Held != nil {
|
||||
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
|
||||
}
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
|
||||
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
|
||||
}
|
||||
}
|
||||
|
||||
m, n = everywhere()
|
||||
delete(n.Haves["g14"], "mail")
|
||||
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
|
||||
t.Error("a node that does not run the module kept the hold")
|
||||
}
|
||||
|
||||
for name, change := range map[string]func(*Module, *Nodes){
|
||||
"set back": func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
},
|
||||
"failing": func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
},
|
||||
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
|
||||
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
|
||||
"not held": func(m *Module, _ *Nodes) { m.Held = false },
|
||||
} {
|
||||
m, n := everywhere()
|
||||
change(&m, &n)
|
||||
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
|
||||
t.Errorf("%s: the hold was lifted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld
|
||||
// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld
|
||||
// (health unknown) and holds no want for it. The canary g14 does not run F.
|
||||
func x11() (Module, Nodes) {
|
||||
m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"},
|
||||
GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld },
|
||||
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
|
||||
n := Nodes{
|
||||
Online: map[string]bool{"g14": true, "shanks": true},
|
||||
Haves: map[string]map[string]delivery.Have{
|
||||
"g14": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
||||
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
||||
"shanks": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
||||
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
||||
},
|
||||
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
|
||||
}
|
||||
return m, n
|
||||
}
|
||||
|
||||
// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by
|
||||
// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that
|
||||
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
|
||||
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
out := Wants(m, n, nil, nil)
|
||||
for _, node := range []string{"g14", "shanks"} {
|
||||
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
|
||||
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
|
||||
}
|
||||
}
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("left on neither list: %+v", out.Neither)
|
||||
}
|
||||
// with the provider healthy, only the canary holds shanks back: still at what it runs
|
||||
m, n = x11()
|
||||
n.Haves["g14"]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
|
||||
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.FP != fOld || w.Rule != delivery.RuleCanary {
|
||||
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
|
||||
}
|
||||
// a node already running F gets its want at F; a removed want is no want either
|
||||
m, n = x11()
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
out = Wants(m, n, map[string]delivery.Want{"g14": {Removed: true}}, nil)
|
||||
if w := out.Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) || w.Removed {
|
||||
t.Errorf("g14 running F with a removed want = %+v, want F", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no
|
||||
// want past the rules, and is named on neither list; the ordering rules still order every move from a want.
|
||||
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
for what, change := range map[string]func(*Module, *Nodes){
|
||||
"an old-form build": func(_ *Module, n *Nodes) {
|
||||
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
},
|
||||
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
|
||||
"no composition": func(m *Module, _ *Nodes) {
|
||||
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, fp == fNew
|
||||
}
|
||||
},
|
||||
} {
|
||||
m, n := x11()
|
||||
change(&m, &n)
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
|
||||
}
|
||||
}
|
||||
m, n := x11()
|
||||
before := map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}, "shanks": {FP: fOld, Composed: dig(fOld)}}
|
||||
out := Wants(m, n, before, nil)
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
|
||||
}
|
||||
for node, w := range out.Wants {
|
||||
if w.FP != fOld {
|
||||
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is
|
||||
// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new
|
||||
// form goes with the node's own composition and is never named.
|
||||
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
m, n := x11()
|
||||
delete(n.Haves["shanks"], "screen-lock")
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
|
||||
}
|
||||
if _, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
|
||||
}
|
||||
m.Built = false
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
||||
}
|
||||
m.FP = ""
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
||||
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
|
||||
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
|
||||
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
|
||||
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
oldForm := func(n Nodes, node string) {
|
||||
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
}
|
||||
providersHealthy := func(n Nodes) {
|
||||
for _, node := range []string{"g14", "shanks"} {
|
||||
n.Haves[node]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}
|
||||
}
|
||||
// the canary g14 runs F healthy: shanks's first want names F
|
||||
m, n := x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w := out.Wants["shanks"]; w.FP != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
|
||||
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
|
||||
}
|
||||
if _, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither)
|
||||
}
|
||||
// the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3
|
||||
m, n = x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
|
||||
}
|
||||
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
|
||||
m, n = x11()
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("providers not healthy: shanks got a want: %+v", w)
|
||||
}
|
||||
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
|
||||
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
|
||||
}
|
||||
// F not granted: nothing, even with the canary healthy
|
||||
m, n = x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
m.Granted = false
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
|
||||
t.Errorf("F not granted: shanks got a want: %+v", w)
|
||||
}
|
||||
// the old-form node is the canary itself: F once its providers run healthy
|
||||
m, n = x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "g14")
|
||||
if w := Wants(m, n, nil, nil).Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("old-form canary: g14's want = %+v, want F", w)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking.
|
||||
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
|
||||
// canary two seconds before the new build was healthy.
|
||||
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
|
||||
for _, phase := range []delivery.Phase{delivery.PhaseBackingUp, delivery.PhaseApplying, delivery.PhaseChecking} {
|
||||
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
||||
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
||||
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
||||
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
||||
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
|
||||
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
|
||||
}
|
||||
}
|
||||
for _, phase := range []delivery.Phase{delivery.PhaseRunning, delivery.PhaseFailed} {
|
||||
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
||||
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
||||
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
||||
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
||||
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
|
||||
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module
|
||||
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
|
||||
// fingerprint, or failed, still does.
|
||||
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
|
||||
setup := func(phase delivery.Phase, fp string, health delivery.Health) (Module, Nodes) {
|
||||
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
|
||||
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]delivery.Have{
|
||||
"ace": {"web": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
||||
"g14": {"web": {FP: fp, Phase: phase, Health: health}}}}
|
||||
return m, n
|
||||
}
|
||||
for _, phase := range []delivery.Phase{delivery.PhaseRemoved, delivery.PhaseHeld, delivery.PhaseApplying, delivery.PhaseBackingUp, delivery.PhaseChecking} {
|
||||
m, n := setup(phase, fOld, delivery.HealthUnknown)
|
||||
if out := Wants(m, n, nil, nil); !out.Lifted {
|
||||
t.Errorf("g14's have in phase %s kept the hold up", phase)
|
||||
}
|
||||
}
|
||||
for what, h := range map[string]delivery.Have{
|
||||
"running another fingerprint": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
"failed": {FP: fNew, Phase: delivery.PhaseFailed, Health: delivery.HealthFailed},
|
||||
} {
|
||||
m, n := setup(h.Phase, h.FP, h.Health)
|
||||
if out := Wants(m, n, nil, nil); out.Lifted {
|
||||
t.Errorf("g14 %s: the hold was lifted", what)
|
||||
}
|
||||
}
|
||||
// a failed health on a removed have holds nothing
|
||||
m, n := setup(delivery.PhaseRemoved, fNew, delivery.HealthFailed)
|
||||
m.Held = false
|
||||
if out := Wants(m, n, nil, nil); out.Held != nil {
|
||||
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
|
||||
}
|
||||
}
|
||||
|
||||
// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a
|
||||
// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes.
|
||||
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
|
||||
m, n := x11()
|
||||
recording(&m)
|
||||
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord}, true
|
||||
}
|
||||
out := Wants(m, n, map[string]delivery.Want{"shanks": {Removed: true}}, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
|
||||
}
|
||||
asked := false
|
||||
for _, r := range out.Record {
|
||||
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == delivery.JudgedRecreates)
|
||||
}
|
||||
if !asked {
|
||||
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the
|
||||
// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on
|
||||
// neither list: a held module is on the held list, whose hold is said and answered on its own.
|
||||
func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
|
||||
for what, change := range map[string]func(*Module, *Nodes){
|
||||
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
|
||||
"held at F, running an old-form build": func(m *Module, n *Nodes) {
|
||||
m.Held, m.Kept = true, true
|
||||
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
},
|
||||
"held by this pass": func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
},
|
||||
} {
|
||||
m, n := x11()
|
||||
change(&m, &n)
|
||||
out := Wants(m, n, nil, nil)
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
|
||||
}
|
||||
if out.Rules["shanks"].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
|
||||
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||
// its three states, and holds a credential of its own.
|
||||
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
func TestManifest(t *testing.T) {
|
||||
m := load(t)
|
||||
if m.Module != "mesh-delivery-assigner" {
|
||||
@@ -32,9 +32,9 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants",
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
@@ -51,9 +51,10 @@ func TestManifest(t *testing.T) {
|
||||
if m.SecretsOwner != "mesh-delivery-assigner" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1622: the assigner puts its compose asks on the Go build seat, as the checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go", m.Uses)
|
||||
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
|
||||
// checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
}
|
||||
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
||||
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
||||
|
||||
@@ -6,16 +6,19 @@
|
||||
"mesh-build-go.steps",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"node-engine.have"
|
||||
],
|
||||
"uses": [
|
||||
"node-build-go"
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "asks", "history": 5 },
|
||||
|
||||
@@ -121,5 +121,12 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifest([]byte(kept.JSON))
|
||||
return parseManifest(kept.JSON)
|
||||
}
|
||||
|
||||
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
|
||||
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
|
||||
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
|
||||
func parseManifest(raw string) (catalogue.Manifest, error) {
|
||||
return catalogue.ParseManifestForInputs([]byte(raw))
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
@@ -93,7 +92,7 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
|
||||
if mk, err := delivery.ManifestsKey(m.Manifest); err == nil {
|
||||
if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found {
|
||||
if kept, err := delivery.DecodeManifest(raw); err == nil {
|
||||
if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil {
|
||||
if man, err := parseManifest(kept.JSON); err == nil && man.Build != nil {
|
||||
for _, a := range man.Build.Artifacts {
|
||||
if a.Language != "" {
|
||||
set[a.Language] = true
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -112,3 +113,31 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t
|
||||
t.Errorf("another repository: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the
|
||||
// catalogue's predefined.json this process never holds: the indexer and the assigner read it with
|
||||
// catalogue.ParseManifestForInputs (issues 1751, 1906). xterm's module.json from mesh-catalog at 835bf22b.
|
||||
func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest := string(raw)
|
||||
data, err := delivery.EncodeManifest(delivery.Manifest{Module: "xterm", JSON: manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := memManifests{delivery.ManifestDigest([]byte(manifest)): data}
|
||||
var said []string
|
||||
r := &repoModules{manifests: store, logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
|
||||
putModule(t, r, "xterm", "novox/mesh-catalog", manifest)
|
||||
r.BuildModules("novox/mesh-catalog")
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "cannot be read") {
|
||||
t.Errorf("the manifest was not read: %s", s)
|
||||
}
|
||||
}
|
||||
if _, err := parseManifest(manifest); err != nil {
|
||||
t.Errorf("manifestReader: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
{
|
||||
"module": "xterm",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"x11-display"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-terminal-emulator",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"open"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"xterm_font",
|
||||
"xterm_colours"
|
||||
],
|
||||
"config": {
|
||||
"face": {
|
||||
"type": "string",
|
||||
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
|
||||
"pattern": "[A-Za-z0-9 ._-]{1,64}",
|
||||
"default": "JetBrainsMono Nerd Font"
|
||||
},
|
||||
"size": {
|
||||
"type": "number",
|
||||
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
|
||||
"minimum": 6,
|
||||
"maximum": 72,
|
||||
"predefined": "font-size",
|
||||
"default": 12
|
||||
},
|
||||
"save-lines": {
|
||||
"type": "number",
|
||||
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
|
||||
"minimum": 0,
|
||||
"maximum": 1000000,
|
||||
"default": 10000
|
||||
},
|
||||
"term-name": {
|
||||
"type": "string",
|
||||
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
|
||||
"pattern": "[A-Za-z0-9._-]{1,64}",
|
||||
"default": "xterm-256color"
|
||||
},
|
||||
"render-font": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
|
||||
"default": true
|
||||
},
|
||||
"bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
|
||||
"default": false
|
||||
},
|
||||
"always-bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
|
||||
"default": false
|
||||
},
|
||||
"allow-bold-fonts": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
|
||||
"default": true
|
||||
},
|
||||
"background": {
|
||||
"type": "string",
|
||||
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"foreground": {
|
||||
"type": "string",
|
||||
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a8a8a8"
|
||||
},
|
||||
"color0": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"color1": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#9e1828"
|
||||
},
|
||||
"color2": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#008800"
|
||||
},
|
||||
"color3": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#d2bb4b"
|
||||
},
|
||||
"color4": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#414171"
|
||||
},
|
||||
"color5": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#963c59"
|
||||
},
|
||||
"color6": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#418179"
|
||||
},
|
||||
"color7": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bebebe"
|
||||
},
|
||||
"color8": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#666666"
|
||||
},
|
||||
"color9": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bc5766"
|
||||
},
|
||||
"color10": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#61a171"
|
||||
},
|
||||
"color11": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#e7db52"
|
||||
},
|
||||
"color12": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#5085af"
|
||||
},
|
||||
"color13": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a97a99"
|
||||
},
|
||||
"color14": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#6ba4a4"
|
||||
},
|
||||
"color15": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#ffffff"
|
||||
},
|
||||
"copy-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> C"
|
||||
},
|
||||
"paste-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> V"
|
||||
}
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "terminal",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "TERMINAL"
|
||||
},
|
||||
"template": "files/env-terminal"
|
||||
},
|
||||
{
|
||||
"collection": "shell:xresources:normal",
|
||||
"name": "xresources-normal",
|
||||
"template": "files/xresources-normal"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "xterm"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/xterm-tools",
|
||||
"binary": "xterm-tools",
|
||||
"loads": [
|
||||
"xterm-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -531,23 +531,98 @@ const MaxWords = 4096
|
||||
|
||||
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
|
||||
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
|
||||
// status line, which is cut short, names it.
|
||||
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
|
||||
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
|
||||
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
|
||||
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
|
||||
func failureWords(f Failure) string {
|
||||
out := strings.TrimSpace(f.Output)
|
||||
if len(out) > MaxWords {
|
||||
out = out[len(out)-MaxWords:]
|
||||
// Start on a whole line, and a whole character.
|
||||
if i := strings.IndexByte(out, '\n'); i >= 0 {
|
||||
out = "... " + strings.TrimLeft(out[i:], "\n")
|
||||
}
|
||||
out = strings.ToValidUTF8(out, "")
|
||||
out, passed := dropSummary(strings.TrimSpace(f.Output))
|
||||
cause := strings.TrimSpace(f.Cause)
|
||||
if cause == "" {
|
||||
cause = causeIn(out)
|
||||
}
|
||||
if c := strings.TrimSpace(f.Cause); c != "" && !strings.HasPrefix(firstLine(out), c) {
|
||||
out = c + "\n" + out
|
||||
switch {
|
||||
case cause == "" && out == "" && passed:
|
||||
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
|
||||
case cause == "" && out == "":
|
||||
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
|
||||
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
|
||||
}
|
||||
if len(cause) > MaxCause {
|
||||
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
|
||||
}
|
||||
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
|
||||
budget := MaxWords
|
||||
if cause != "" {
|
||||
budget -= len(cause) + 1
|
||||
}
|
||||
if len(out) > budget {
|
||||
const marker = "... "
|
||||
tail := out[len(out)-(budget-len(marker)):]
|
||||
// Start on a whole line, and a whole character.
|
||||
if i := strings.IndexByte(tail, '\n'); i >= 0 {
|
||||
tail = strings.TrimLeft(tail[i:], "\n")
|
||||
}
|
||||
out = marker + strings.ToValidUTF8(tail, "")
|
||||
}
|
||||
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
|
||||
out = strings.TrimRight(cause+"\n"+out, "\n")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
|
||||
// push the words past MaxWords).
|
||||
const MaxCause = 1024
|
||||
|
||||
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
|
||||
// whether a "PASS" was among them: the package's tests passed before it failed.
|
||||
func dropSummary(out string) (string, bool) {
|
||||
passed := false
|
||||
lines := strings.Split(out, "\n")
|
||||
kept := lines[:0]
|
||||
for _, l := range lines {
|
||||
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
|
||||
switch t := strings.TrimRight(l, "\r"); {
|
||||
case t == "PASS":
|
||||
passed = true
|
||||
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
|
||||
default:
|
||||
kept = append(kept, l)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(kept, "\n")), passed
|
||||
}
|
||||
|
||||
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
|
||||
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
|
||||
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
|
||||
// line holding the same words is not the cause (review round 1).
|
||||
func causeIn(out string) string {
|
||||
lines := strings.Split(out, "\n")
|
||||
for i, l := range lines {
|
||||
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
|
||||
continue
|
||||
}
|
||||
words := "WARNING: DATA RACE"
|
||||
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
|
||||
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
|
||||
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
|
||||
words += " in " + strings.TrimSpace(lines[i+2])
|
||||
}
|
||||
}
|
||||
return words
|
||||
}
|
||||
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
|
||||
for _, l := range lines {
|
||||
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
|
||||
return l
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// firstLine is the first non-empty line of a failure's words.
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What the build module before this fix sent for two packages that failed after their tests passed (issue 1901, its third
|
||||
// meeting: a try of mesh-controller kept "PASS" as its failure's words). Each is that build module's Output for the real
|
||||
// `go test -json` of a throwaway package (mesh-catalog modules/mesh-build-go testdata/captured/go-test-json-pkg-*.jsonl):
|
||||
// TestMain panicking in a goroutine after m.Run, TestMain exiting 2 after m.Run, and TestMain racing after m.Run.
|
||||
const (
|
||||
oldPassThenPanic = "PASS\npanic: the bus closed after the tests: nats: connection closed\ngoroutine 20 [running]:\n" +
|
||||
"example.com/passthenfail.TestMain.func1()\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||
"created by example.com/passthenfail.TestMain in goroutine 1\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||
"FAIL\texample.com/passthenfail\t0.005s"
|
||||
oldPassThenExit = "PASS\nFAIL\texample.com/mainexit\t0.004s"
|
||||
// TestMain racing after m.Run passed, under go test -race (go-test-json-pkg-racemain.jsonl).
|
||||
oldPassThenRace = "PASS\n==================\nWARNING: DATA RACE\nWrite at 0x000000836428 by goroutine 9:\n example.com/racemain.TestMain.func1()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x24\nPrevious write at 0x000000836428 by main goroutine:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:16 +0x44\n main.main()\n _testmain.go:48 +0x171\nGoroutine 9 (running) created at:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x37\n main.main()\n _testmain.go:48 +0x171\n==================\nFound 1 data race(s)\nFAIL\texample.com/racemain\t1.031s"
|
||||
)
|
||||
|
||||
// A failure with no cause told apart never has "PASS" or "ok" as its words: the checker takes the panic line from the
|
||||
// output, or says that the tests passed and the program failed after them (1901/AC5).
|
||||
func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
|
||||
cases := []struct {
|
||||
name, output, want string
|
||||
}{
|
||||
{"panic after the tests", oldPassThenPanic, "panic: the bus closed after the tests: nats: connection closed"},
|
||||
{"exit after the tests", oldPassThenExit, "the tests passed, then the test program exited with a failure"},
|
||||
{"race after the tests", oldPassThenRace, "WARNING: DATA RACE: Write at 0x000000836428 by goroutine 9 in example.com/racemain.TestMain.func1()"},
|
||||
{"a panic deep in a long output", "PASS\npanic: the seat row vanished\n" + strings.Repeat("goroutine 7 [chan receive]:\n\tsome/frame.go:1 +0x1\n", 300) + "FAIL\tp\t2.0s", "panic: the seat row vanished"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
|
||||
for _, l := range strings.Split(v.Failed.Words, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "PASS" || strings.HasPrefix(l, "ok ") || strings.HasPrefix(l, "ok\t") {
|
||||
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(v.Failed.Words, c.want) {
|
||||
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
|
||||
}
|
||||
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
|
||||
t.Errorf("1901/AC5: status = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A cause the build module told apart is cut to a line's bound, so the words keep their bound (the 1901 review's note).
|
||||
func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
|
||||
f := Failure{Package: "p", Test: "TestX", Cause: strings.Repeat("c", 5000), Output: "x_test.go:1: something"}
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||
if len(v.Failed.Words) > MaxWords+200 {
|
||||
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
|
||||
}
|
||||
}
|
||||
|
||||
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as only go test's last line, with no
|
||||
// cause: the words then say so, never nothing (review round 1). Each output is what that build module sends for the
|
||||
// capture of the same name (go-test-json-pkg-timeout, -exiting, -killed): the package's own output, its "FAIL" line.
|
||||
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
|
||||
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
|
||||
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
|
||||
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
|
||||
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A line of the test's own that starts as go's summary does, or that holds a runtime's words indented, is not taken for
|
||||
// either (review round 1): go prints its summary in exact forms and the runtime's lines at column 0.
|
||||
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
|
||||
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
|
||||
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
|
||||
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
|
||||
// The words keep MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
|
||||
func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
|
||||
for _, cause := range []string{"", "a cause", strings.Repeat("c", 3000)} {
|
||||
for _, n := range []int{100, 170, 171, 172, 500} {
|
||||
f := Failure{Package: "p", Test: "TestX", Cause: cause, Output: strings.Repeat("a log line of twenty\n", n) + "the end"}
|
||||
if w := failureWords(f); len(w) > MaxWords {
|
||||
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), MaxWords)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's asks reach the operator (issue 1906). The assigner writes its asks (the hold of a fingerprint, the restore
|
||||
// of a data module, the backup before an apply and a move of a module of policy record) in its bucket asks; until now the
|
||||
// teller only filled them from an answer given in the view, so an ask nobody opened the view for reached nobody: on
|
||||
// 2026-10-25 seven modules stayed held on every machine and the operator was never asked. Now the teller raises each on the
|
||||
// operator channel as it raises the gate's merge asks: as the asker mesh-delivery-teller, once per router id, through the same
|
||||
// records, the same wait for room and the same warrants. The operator's answer, from the phone or from the view, is written
|
||||
// under the ask's own key in the bucket answers, which the assigner reads (mesh-delivery-assigner pass.go answeredWith); none
|
||||
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
|
||||
|
||||
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
|
||||
// asks.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
|
||||
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
|
||||
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
|
||||
const mostAssignerOpen = asks.MostOpen - 1
|
||||
|
||||
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
|
||||
const assignerOpenRecord = "assigner.open"
|
||||
|
||||
// assignerOpen is the router ids of the assigner's asks open now: those asked whose ask is neither answered nor ended. An id
|
||||
// that ended is dropped from the record as it is read.
|
||||
func (t *Teller) assignerOpen() ([]string, *stdio.StateEntry) {
|
||||
e, err := t.Store.Get(assignerOpenRecord)
|
||||
if err != nil || e == nil {
|
||||
return nil, e
|
||||
}
|
||||
var ids []string
|
||||
if err := json.Unmarshal(e.Value, &ids); err != nil {
|
||||
t.logf("the record %s cannot be read: %v", assignerOpenRecord, err)
|
||||
return nil, e
|
||||
}
|
||||
return slices.DeleteFunc(ids, func(id string) bool {
|
||||
k, _, err := t.keptOf(id)
|
||||
return err == nil && !open(k)
|
||||
}), e
|
||||
}
|
||||
|
||||
// assignerRoom says whether another of the assigner's asks may be asked now.
|
||||
func (t *Teller) assignerRoom() bool {
|
||||
ids, _ := t.assignerOpen()
|
||||
return len(ids) < mostAssignerOpen
|
||||
}
|
||||
|
||||
// assignerAsked keeps that the assigner's ask with this router id is asked.
|
||||
func (t *Teller) assignerAsked(id string) {
|
||||
ids, e := t.assignerOpen()
|
||||
if !slices.Contains(ids, id) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
var err error
|
||||
if e == nil {
|
||||
_, err = t.Store.Create(assignerOpenRecord, ids)
|
||||
} else {
|
||||
_, err = t.Store.Update(assignerOpenRecord, ids, e.Revision)
|
||||
}
|
||||
if err != nil {
|
||||
t.logf("the record %s could not be kept: %v", assignerOpenRecord, err)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForAssignerRoom puts an assigner ask in the wait for room, once.
|
||||
func (t *Teller) waitForAssignerRoom(key, id string) {
|
||||
ws, rec, err := t.waiting()
|
||||
if err != nil {
|
||||
t.logf("%v", err)
|
||||
return
|
||||
}
|
||||
if slices.ContainsFunc(ws, func(x waitingAsk) bool { return x.ID == id }) {
|
||||
return
|
||||
}
|
||||
if t.keepWaiting(append(ws, waitingAsk{Key: key, ID: id}), rec) == nil {
|
||||
t.logf("%s: the assigner's ask %s waits: %d of the assigner's asks are open, the most the teller keeps open so a merge ask always has room", key, id, mostAssignerOpen)
|
||||
}
|
||||
}
|
||||
|
||||
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
|
||||
func isAssignerKey(key string) bool {
|
||||
k, err := delivery.ParseAnyAskKey(key)
|
||||
return err == nil && k.Kind != delivery.AskKindMerge
|
||||
}
|
||||
|
||||
// assignerAsk reads the assigner's ask at key: found false when the key holds none.
|
||||
func (t *Teller) assignerAsk(key string) (delivery.AssignerAsk, bool, error) {
|
||||
if t.Assigner == nil {
|
||||
return delivery.AssignerAsk{}, false, fmt.Errorf("the teller reads no assigner asks")
|
||||
}
|
||||
raw, found, err := t.Assigner.Get(key)
|
||||
if err != nil || !found {
|
||||
return delivery.AssignerAsk{}, false, err
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(key, raw)
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, false, fmt.Errorf("the assigner's key %s holds no ask: %w", key, err)
|
||||
}
|
||||
return a, true, nil
|
||||
}
|
||||
|
||||
// raiseAssignerAsk asks the operator the assigner's ask at key, once per router id: an ask answered already (from the view or
|
||||
// the phone) is not asked; an earlier asking still open is taken back first; an asking already made is only caught up with
|
||||
// at the router, and one that ended is not asked again, for the router drops a second ask under an id it keeps.
|
||||
func (t *Teller) raiseAssignerAsk(key string) {
|
||||
a, found, err := t.assignerAsk(key)
|
||||
if err != nil {
|
||||
t.logf("%s: the ask is not raised: %v", key, err)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
id := a.Ask.ID
|
||||
if raw, _, has, err := t.Answers.Get(delivery.AnswerKey(key)); err != nil {
|
||||
t.logf("%s: the answers could not be read, so the ask is not raised: %v", key, err)
|
||||
return
|
||||
} else if has && answeredAsking(raw) == id {
|
||||
return
|
||||
}
|
||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
return
|
||||
}
|
||||
if grec != nil && prior.ID != id {
|
||||
t.cancelOpen(key, prior.ID) // a new asking of the same key: the earlier one is taken back
|
||||
}
|
||||
k, _, err := t.keptOf(id)
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
return
|
||||
}
|
||||
if k != nil {
|
||||
if grec == nil || prior.ID != id {
|
||||
_ = t.note(id, key, 0, grec)
|
||||
}
|
||||
if open(k) {
|
||||
t.catchUp(id)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := t.note(id, key, 0, grec); err != nil {
|
||||
t.logf("%s: the record of the ask %s could not be kept, so it is not raised: %v", key, id, err)
|
||||
return
|
||||
}
|
||||
if !t.assignerRoom() {
|
||||
t.waitForAssignerRoom(key, id)
|
||||
return
|
||||
}
|
||||
if err := t.Asker.Ask(a.Ask); err != nil {
|
||||
t.logf("%s: the ask %s was not asked: %v", key, id, err)
|
||||
if kept, _, _ := t.keptOf(id); open(kept) {
|
||||
_ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it
|
||||
}
|
||||
return
|
||||
}
|
||||
t.assignerAsked(id)
|
||||
t.logf("%s: the assigner's ask %s is asked of the operator (%s)", key, id, a.Ask.Headline)
|
||||
}
|
||||
|
||||
// OnAssignerAskGone handles a key the assigner took away from its asks (a hold it lifted by itself): the open router ask of
|
||||
// its latest asking is taken back, so the operator is not asked about what no longer holds.
|
||||
func (t *Teller) OnAssignerAskGone(key string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if !isAssignerKey(key) {
|
||||
t.logf("the assigner's key %s was taken away and is no ask key of the assigner, and is left", key)
|
||||
return
|
||||
}
|
||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||
if err != nil || grec == nil {
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.cancelOpen(key, prior.ID)
|
||||
}
|
||||
|
||||
// answerAssigner is the act of a warrant for one of the assigner's asks: the ask must still stand at its key under the asking
|
||||
// and the words the operator answered, and then the answer is written under the ask's key, once. Its options bind no act:
|
||||
// the assigner reads the option chosen (asker.Client.Take has checked the option, its level and the ask's digest).
|
||||
func (t *Teller) answerAssigner(key string, o asks.Option, w asks.Warrant) error {
|
||||
a, found, err := t.assignerAsk(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
t.logf("%s: the ask %s was answered and the assigner has taken it away: the answer changes nothing", key, w.Ask)
|
||||
return nil
|
||||
}
|
||||
if a.Ask.ID != w.Ask {
|
||||
t.logf("%s: the ask %s was answered, and the assigner has asked again under %s: an answer to an earlier asking changes nothing", key, w.Ask, a.Ask.ID)
|
||||
return nil
|
||||
}
|
||||
if a.Ask.Digest() != w.AskDigest {
|
||||
return fmt.Errorf("the assigner's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)", key, a.Ask.Digest(), w.AskDigest)
|
||||
}
|
||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.writeAnswer(key, w.Ask, body, w.Says())
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
|
||||
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
|
||||
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
b.t.OnAssignerAsk(key)
|
||||
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
|
||||
}
|
||||
var got asks.Ask
|
||||
if err := json.Unmarshal(asked[0].body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.ID != ask.ID || got.Digest() != ask.Digest() {
|
||||
t.Errorf("the router's ask %+v differs from the assigner's", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: without the view's bucket the assigner's asks are still raised.
|
||||
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner := memGate{}
|
||||
b.t.Assigner = assigner
|
||||
key, value, _ := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
|
||||
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
|
||||
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
|
||||
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
w := ackFor(ask, "release")
|
||||
b.deliver(t, w)
|
||||
raw, ok := b.answers.written[delivery.AnswerKey(key)]
|
||||
if !ok {
|
||||
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
|
||||
}
|
||||
if a, err := delivery.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
|
||||
t.Errorf("answer = %+v, %v", a, err)
|
||||
}
|
||||
b.deliver(t, w)
|
||||
if b.answers.writes != 1 || len(b.conds) != 0 {
|
||||
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
|
||||
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
delete(assigner, key)
|
||||
b.deliver(t, ackFor(ask, "release"))
|
||||
if b.answers.writes != 0 {
|
||||
t.Errorf("an answer to a withdrawn ask was written")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
|
||||
// itself) is taken back too; an ask answered before it is heard is never raised.
|
||||
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
|
||||
cancelled := func(b *bed, id string) bool {
|
||||
for _, p := range b.askedOf(asks.CancelSubject(delivery.TellerModule)) {
|
||||
if string(p.body) == `{"id":"`+id+`"}` {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
b := newBed(t)
|
||||
assigner, view := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
view[key+".view"] = viewAnswer(t, "release", "")
|
||||
b.t.OnViewAnswer(key + ".view")
|
||||
if !cancelled(b, ask.ID) {
|
||||
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
|
||||
}
|
||||
|
||||
b = newBed(t)
|
||||
assigner, _ = withView(b)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
delete(assigner, key)
|
||||
b.t.OnAssignerAskGone(key)
|
||||
if !cancelled(b, ask.ID) {
|
||||
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
|
||||
}
|
||||
|
||||
b = newBed(t)
|
||||
assigner, view = withView(b)
|
||||
assigner[key] = value
|
||||
view[key+".view"] = viewAnswer(t, "keep", "")
|
||||
b.t.OnAssignerAsk(key)
|
||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 0 {
|
||||
t.Errorf("an ask answered in the view was raised %d times", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
|
||||
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
|
||||
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
_, first := b.askOf(t, 1, 1)
|
||||
b.askOf(t, 2, 2)
|
||||
b.askOf(t, 3, 3)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
b.refuse(t, ask.ID, roomRefusal)
|
||||
b.deliver(t, warrantFor(first.Ask, "approve"))
|
||||
got := b.askedIDs()
|
||||
if len(got) != 5 || got[4] != ask.ID {
|
||||
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
|
||||
}
|
||||
|
||||
b = newBed(t)
|
||||
assigner, _ = withView(b)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
b.refuse(t, ask.ID, "the ask has no headline")
|
||||
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
|
||||
t.Errorf("conditions = %+v", b.conds)
|
||||
}
|
||||
}
|
||||
|
||||
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
|
||||
func ackFor(a asks.Ask, option string) asks.Warrant {
|
||||
w := warrantFor(a, option)
|
||||
w.Level, w.Label = asks.Acknowledge, option
|
||||
return w
|
||||
}
|
||||
|
||||
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
|
||||
func holdAskOf(t *testing.T, n int) (string, []byte, asks.Ask) {
|
||||
t.Helper()
|
||||
key, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, ask := holdAsk(t)
|
||||
ask.ID = delivery.AskID(key)
|
||||
value, err := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value, ask
|
||||
}
|
||||
|
||||
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
|
||||
// ask is still asked, and each hold answered lets the next waiting one be asked.
|
||||
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
var held []asks.Ask
|
||||
for n := 0; n < 7; n++ {
|
||||
key, value, ask := holdAskOf(t, n)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
held = append(held, ask)
|
||||
}
|
||||
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
|
||||
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
|
||||
}
|
||||
_, merge := b.askOf(t, 1, 1)
|
||||
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
|
||||
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
|
||||
}
|
||||
if len(b.dropped) != 0 || len(b.conds) != 0 {
|
||||
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
|
||||
}
|
||||
b.deliver(t, ackFor(held[0], "release"))
|
||||
got := b.askedIDs()
|
||||
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
|
||||
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
|
||||
}
|
||||
// The merge ask answered frees its own place, and no hold takes it.
|
||||
b.deliver(t, warrantFor(merge.Ask, "approve"))
|
||||
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
|
||||
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
|
||||
}
|
||||
}
|
||||
@@ -524,8 +524,9 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
||||
t.Fatalf("answer %+v err %v", a, err)
|
||||
}
|
||||
// The view's answer written again changes nothing under the ask.
|
||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "keep", "")); err != nil {
|
||||
// The view's answer written again with the same option changes nothing under the ask (another option, later, is the
|
||||
// operator changing their mind: TestTellerTakesAChangedHoldAnswerFromTheView, issue 1906 note 23).
|
||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
@@ -545,3 +546,148 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
||||
t.Error("the merge's ask was answered from a view key")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer starts the program against the bus: the assigner's hold ask, put in
|
||||
// its bucket, is on the operator channel's queue as the teller's ask; the router's warrant for "release" is written under the
|
||||
// ask's key, where the assigner reads it; a second ask the assigner takes away (a hold it lifted) is cancelled at the router
|
||||
// (1906/H1, H2 on the real bus).
|
||||
func TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer(t *testing.T) {
|
||||
now = time.Now().UTC()
|
||||
b := startFull(t)
|
||||
assigner := b.as(t, "n1.mesh-delivery-assigner")
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
logs := &syncBuffer{}
|
||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||
|
||||
key, value, ask := holdAsk(t)
|
||||
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond) // the program hears warrants and watches
|
||||
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raw *jetstream.RawStreamMsg
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw == nil {
|
||||
t.Fatalf("the assigner's ask is not on the seat's queue: %v\n%s", err, logs.String())
|
||||
}
|
||||
var asked asks.Ask
|
||||
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.ID != ask.ID || asked.Digest() != ask.Digest() {
|
||||
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
|
||||
}
|
||||
w, _ := json.Marshal(ackFor(ask, "release"))
|
||||
if err := b.ctl.Publish(seatDecided, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var e jetstream.KeyValueEntry
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if e == nil {
|
||||
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
|
||||
}
|
||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
||||
t.Fatalf("answer %+v err %v", a, err)
|
||||
}
|
||||
|
||||
key2, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask2 := ask
|
||||
ask2.ID = delivery.AskID(key2)
|
||||
value2, err := delivery.EncodeAssignerAsk(key2, delivery.AssignerAsk{Ask: ask2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := askKV.Put(b.ctx, key2, value2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if strings.Contains(logs.String(), ask2.ID+" is asked") {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := askKV.Delete(b.ctx, key2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
|
||||
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestTellerTakesAChangedHoldAnswerFromTheView starts the program against the bus: a hold ask answered "keep" earlier, and the
|
||||
// view's answer "release" written now, replace the answer of that asking with "release" (1906/N23 on the real bus).
|
||||
func TestTellerTakesAChangedHoldAnswerFromTheView(t *testing.T) {
|
||||
now = time.Now().UTC()
|
||||
b := startFull(t)
|
||||
assigner, view := b.as(t, "n1.mesh-delivery-assigner"), b.as(t, "n1.mesh-delivery-view")
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, value, ask := holdAsk(t)
|
||||
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator chose keep"})
|
||||
if _, err := answers.Put(b.ctx, delivery.AnswerKey(key), kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
logs := &syncBuffer{}
|
||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||
viewKV, err := view.KeyValue(b.ctx, viewAnswers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond) // the program is watching
|
||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "it was a flake")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
||||
if a, err := delivery.DecodeAnswer(e.Value()); err == nil && a.Option == "release" && answeredAsking(e.Value()) == ask.ID {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the kept hold's answer was not changed to release\n%s", logs.String())
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -107,3 +108,15 @@ func (g kvGate) Get(key string) ([]byte, bool, error) {
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}
|
||||
|
||||
// GetAt is Get with the time the value was written.
|
||||
func (g kvGate) GetAt(key string) ([]byte, time.Time, bool, error) {
|
||||
e, err := g.kv.Get(g.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, time.Time{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, time.Time{}, false, err
|
||||
}
|
||||
return e.Value(), e.Created(), true, nil
|
||||
}
|
||||
|
||||
@@ -141,14 +141,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
||||
failed := make(chan error, 2)
|
||||
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
||||
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
|
||||
if viewKV != nil {
|
||||
t.View = kvGate{ctx: ctx, kv: viewKV}
|
||||
}
|
||||
if assignerKV != nil {
|
||||
t.Assigner = kvGate{ctx: ctx, kv: assignerKV}
|
||||
go func() {
|
||||
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil)
|
||||
}()
|
||||
}
|
||||
if assignerKV != nil && viewKV != nil {
|
||||
t.Assigner, t.View = kvGate{ctx: ctx, kv: assignerKV}, kvGate{ctx: ctx, kv: viewKV}
|
||||
go func() {
|
||||
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
|
||||
}()
|
||||
go func() {
|
||||
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, func(string) {}, nil)
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
|
||||
@@ -170,7 +176,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
logger.Printf("the bucket %s cannot be read (%v): the view's answers fill no ask until the teller is started again with it", name, err)
|
||||
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
|
||||
return nil
|
||||
}
|
||||
return kv
|
||||
|
||||
@@ -340,6 +340,9 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
||||
if srec == nil {
|
||||
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
|
||||
}
|
||||
if isAssignerKey(s.Key) {
|
||||
return t.answerAssigner(s.Key, o, w)
|
||||
}
|
||||
raw, found, err := t.Gate.Get(s.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
|
||||
@@ -367,26 +370,31 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Compare-and-set: created when the key has no answer; updated at the revision read when it holds an earlier asking's
|
||||
// answer (the gate ignores that one); left when it holds this asking's.
|
||||
akey := delivery.AnswerKey(s.Key)
|
||||
return t.writeAnswer(s.Key, w.Ask, body, w.Says())
|
||||
}
|
||||
|
||||
// writeAnswer writes the answer to the asking askID under the ask's key by compare-and-set: created when the key has no answer;
|
||||
// updated at the revision read when it holds an earlier asking's answer (the reader ignores that one); left when it holds this
|
||||
// asking's.
|
||||
func (t *Teller) writeAnswer(key, askID string, body []byte, words string) error {
|
||||
akey := delivery.AnswerKey(key)
|
||||
cur, crev, found, err := t.Answers.Get(akey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the answer under %s could not be read: %w", s.Key, err)
|
||||
return fmt.Errorf("the answer under %s could not be read: %w", key, err)
|
||||
}
|
||||
if found && answeredAsking(cur) == w.Ask {
|
||||
t.logf("%s: an answer is there already, and is not written again", s.Key)
|
||||
if found && answeredAsking(cur) == askID {
|
||||
t.logf("%s: an answer is there already, and is not written again", key)
|
||||
return nil
|
||||
}
|
||||
written, err := t.Answers.Write(akey, body, crev, found)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the answer under %s could not be written: %w", s.Key, err)
|
||||
return fmt.Errorf("the answer under %s could not be written: %w", key, err)
|
||||
}
|
||||
if !written {
|
||||
t.logf("%s: another writer answered first, and the answer is not written again", s.Key)
|
||||
t.logf("%s: another writer answered first, and the answer is not written again", key)
|
||||
return nil
|
||||
}
|
||||
t.logf("%s: %s", s.Key, w.Says())
|
||||
t.logf("%s: %s", key, words)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -423,10 +431,10 @@ func (t *Teller) saidOnce(h asker.Handled, w asks.Warrant, err error) {
|
||||
t.clear(w.Ask)
|
||||
case h == asker.Refused || (h == asker.Acted && err != nil):
|
||||
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
|
||||
t.raise(w.Ask, "A merge Approve did not hold", "its answer did not hold here: "+oneLine(err))
|
||||
t.raise(w.Ask, t.kindOf(w.Ask).answerHeadline, "its answer did not hold here: "+oneLine(err))
|
||||
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
|
||||
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
|
||||
t.raise(w.Ask, "A merge ask was refused", "the router refused it: "+w.Words)
|
||||
t.raise(w.Ask, t.kindOf(w.Ask).refusedHeadline, "the router refused it: "+w.Words)
|
||||
case err != nil:
|
||||
t.logf("the warrant of %s: %v", w.Ask, err)
|
||||
}
|
||||
@@ -446,6 +454,30 @@ func oneLine(err error) string {
|
||||
// conditionKey is the key of the condition about an ask.
|
||||
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
|
||||
|
||||
// askWords are the words the teller's conditions use for one kind of ask: the gate's merge ask, or one of the assigner's.
|
||||
type askWords struct {
|
||||
scope, noun, needs, nothing, resolved string
|
||||
answerHeadline, refusedHeadline, unaskedHeadline string
|
||||
}
|
||||
|
||||
var (
|
||||
mergeWords = askWords{scope: "merge", noun: "The merge ask ", needs: "to ask for the Approve again", nothing: "Nothing was merged.",
|
||||
resolved: "A good answer to the merge ask was written.", answerHeadline: "A merge Approve did not hold",
|
||||
refusedHeadline: "A merge ask was refused", unaskedHeadline: "A merge ask could not be asked"}
|
||||
assignerWords = askWords{scope: "delivery", noun: "The delivery ask ", needs: "to answer it in the delivery view",
|
||||
nothing: "Nothing was changed on a machine.", resolved: "A good answer to the delivery ask was written.",
|
||||
answerHeadline: "An answer to a delivery ask did not hold", refusedHeadline: "A delivery ask was refused",
|
||||
unaskedHeadline: "A delivery ask could not be asked"}
|
||||
)
|
||||
|
||||
// kindOf is the words for the ask with this router id, by the key it was asked for.
|
||||
func (t *Teller) kindOf(askID string) askWords {
|
||||
if isAssignerKey(t.condID(askID)) {
|
||||
return assignerWords
|
||||
}
|
||||
return mergeWords
|
||||
}
|
||||
|
||||
func (t *Teller) raise(askID, headline, why string) {
|
||||
if t.Say == nil {
|
||||
return
|
||||
@@ -457,11 +489,12 @@ func (t *Teller) raise(askID, headline, why string) {
|
||||
}
|
||||
t.logf("the condition of %s could not be kept: %v", id, err)
|
||||
}
|
||||
needs := "to ask for the Approve again"
|
||||
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateRaised,
|
||||
Summary: "The merge ask " + id + ": " + why, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". The merge ask " + id + ", " + why + ". Nothing was merged.",
|
||||
Resolved: "A good answer to the merge ask was written."}
|
||||
k := t.kindOf(askID)
|
||||
needs := k.needs
|
||||
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: k.scope, ID: id}, State: asks.StateRaised,
|
||||
Summary: k.noun + id + ": " + why, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". " + k.noun + id + ", " + why + ". " + k.nothing,
|
||||
Resolved: k.resolved}
|
||||
if err := t.Say(c); err != nil {
|
||||
t.logf("the condition of %s could not be said: %v", id, err)
|
||||
_ = t.Store.Delete(condKey(id))
|
||||
@@ -477,7 +510,7 @@ func (t *Teller) clear(askID string) {
|
||||
if err != nil || e == nil {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateCleared}
|
||||
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: t.kindOf(askID).scope, ID: id}, State: asks.StateCleared}
|
||||
if err := t.Say(c); err != nil {
|
||||
t.logf("the condition of %s could not be cleared: %v", id, err)
|
||||
return
|
||||
@@ -578,7 +611,7 @@ func (t *Teller) roomRefused(w asks.Warrant) {
|
||||
return
|
||||
}
|
||||
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
|
||||
t.raise(w.Ask, "A merge ask could not be asked", fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
||||
t.raise(w.Ask, t.kindOf(w.Ask).unaskedHeadline, fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
||||
return
|
||||
}
|
||||
if t.keepWaiting(ws, rec) != nil {
|
||||
@@ -675,6 +708,11 @@ func (t *Teller) askWaiting(n int) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
assigner := isAssignerKey(x.Key)
|
||||
if assigner && !t.assignerRoom() {
|
||||
i++ // the assigner's asks hold their most open: it waits, and a merge ask after it may be asked
|
||||
continue
|
||||
}
|
||||
a, why := t.stillAsked(x)
|
||||
if why != "" {
|
||||
t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why)
|
||||
@@ -687,6 +725,9 @@ func (t *Teller) askWaiting(n int) {
|
||||
return
|
||||
}
|
||||
ws[i].Asked = true
|
||||
if assigner {
|
||||
t.assignerAsked(x.ID)
|
||||
}
|
||||
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
|
||||
if n--; n <= 0 {
|
||||
return
|
||||
@@ -701,6 +742,9 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
||||
if x.Key == "" {
|
||||
return asks.Ask{}, "the teller holds no gate key for it"
|
||||
}
|
||||
if isAssignerKey(x.Key) {
|
||||
return t.stillAskedByAssigner(x)
|
||||
}
|
||||
raw, found, err := t.Gate.Get(x.Key)
|
||||
if err != nil {
|
||||
return asks.Ask{}, "the gate's ask could not be read: " + err.Error()
|
||||
@@ -724,6 +768,26 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
||||
return ga.Ask, ""
|
||||
}
|
||||
|
||||
// stillAskedByAssigner is stillAsked for one of the assigner's asks.
|
||||
func (t *Teller) stillAskedByAssigner(x waitingAsk) (asks.Ask, string) {
|
||||
a, found, err := t.assignerAsk(x.Key)
|
||||
switch {
|
||||
case err != nil:
|
||||
return asks.Ask{}, err.Error()
|
||||
case !found:
|
||||
return asks.Ask{}, "the assigner has taken it away"
|
||||
case a.Ask.ID != x.ID:
|
||||
return asks.Ask{}, "the assigner asks again under " + a.Ask.ID
|
||||
}
|
||||
if cur, _, has, err := t.Answers.Get(delivery.AnswerKey(x.Key)); err == nil && has && answeredAsking(cur) == x.ID {
|
||||
return asks.Ask{}, "it is answered"
|
||||
}
|
||||
if err := a.Ask.Check(t.now()); err != nil {
|
||||
return asks.Ask{}, "it no longer holds: " + err.Error()
|
||||
}
|
||||
return a.Ask, ""
|
||||
}
|
||||
|
||||
// askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and
|
||||
// publishes it under the bus message id given.
|
||||
func (t *Teller) askAgain(a asks.Ask, msgID string) error {
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -25,7 +26,8 @@ func (t *Teller) OnViewAnswer(key string) {
|
||||
t.fillFromView(askKey)
|
||||
}
|
||||
|
||||
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now.
|
||||
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now, and an
|
||||
// ask not answered yet is raised on the operator channel (issue 1906; assignerasks.go).
|
||||
func (t *Teller) OnAssignerAsk(key string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
@@ -34,10 +36,14 @@ func (t *Teller) OnAssignerAsk(key string) {
|
||||
return
|
||||
}
|
||||
t.fillFromView(key)
|
||||
t.raiseAssignerAsk(key)
|
||||
}
|
||||
|
||||
// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the
|
||||
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet.
|
||||
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet. A hold ask answered
|
||||
// already takes a later answer of the view that chose another option: the operator changed their mind (issue 1906 note 23: a
|
||||
// "keep" given on the phone could not be undone). Later means the view's answer was written after the answer that stands, so a
|
||||
// view answer read again at a start never replaces a newer answer from the phone.
|
||||
func (t *Teller) fillFromView(askKey string) {
|
||||
if t.Assigner == nil || t.View == nil {
|
||||
return
|
||||
@@ -47,7 +53,7 @@ func (t *Teller) fillFromView(askKey string) {
|
||||
t.logf("%s is not answered from the view: %v", askKey, err)
|
||||
return
|
||||
}
|
||||
rawAnswer, found, err := t.View.Get(viewKey)
|
||||
rawAnswer, at, found, err := t.viewAnswer(viewKey)
|
||||
if err != nil {
|
||||
t.logf("%s: the view's answer could not be read, so the ask is left: %v", askKey, err)
|
||||
return
|
||||
@@ -83,14 +89,14 @@ func (t *Teller) fillFromView(askKey string) {
|
||||
t.logf("%s: the answer could not be read: %v", askKey, err)
|
||||
return
|
||||
}
|
||||
if has && answeredAsking(cur) == ask.Ask.ID {
|
||||
if has && answeredAsking(cur) == ask.Ask.ID && !t.changesMind(askKey, cur, va, at) {
|
||||
return // this asking is answered, from the view or elsewhere
|
||||
}
|
||||
words := "answered in the view (mesh-delivery-view): " + va.Option
|
||||
if w := strings.TrimSpace(va.Words); w != "" {
|
||||
words += ": " + w
|
||||
}
|
||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + t.now().UTC().Format("2006-01-02T15:04:05.999999999Z07:00"), Words: words})
|
||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + at.UTC().Format(time.RFC3339Nano), Words: words})
|
||||
if err != nil {
|
||||
t.logf("%s: the answer could not be made: %v", askKey, err)
|
||||
return
|
||||
@@ -103,5 +109,48 @@ func (t *Teller) fillFromView(askKey string) {
|
||||
t.logf("%s: another writer answered first, and the view's answer is not written", askKey)
|
||||
default:
|
||||
t.logf("%s: %s", askKey, words)
|
||||
t.cancelOpen(askKey, ask.Ask.ID) // answered in the view: the same ask on the phone is taken back
|
||||
}
|
||||
}
|
||||
|
||||
// timedAnswers is the view's bucket as the program reads it: each answer with the time it was written.
|
||||
type timedAnswers interface {
|
||||
GetAt(key string) (value []byte, at time.Time, found bool, err error)
|
||||
}
|
||||
|
||||
// viewAnswer is the view's answer at key and when it was written; a bucket that keeps no time gives the teller's now.
|
||||
func (t *Teller) viewAnswer(key string) ([]byte, time.Time, bool, error) {
|
||||
if tv, ok := t.View.(timedAnswers); ok {
|
||||
v, at, found, err := tv.GetAt(key)
|
||||
if err == nil && !at.IsZero() {
|
||||
return v, at, found, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, time.Time{}, false, err
|
||||
}
|
||||
return v, t.now(), found, nil
|
||||
}
|
||||
v, found, err := t.View.Get(key)
|
||||
return v, t.now(), found, err
|
||||
}
|
||||
|
||||
// changesMind says whether the view's answer va, written at at, replaces the answer cur of the same asking of the hold ask at
|
||||
// askKey: it chose another option and was written after cur's warrant.
|
||||
func (t *Teller) changesMind(askKey string, cur []byte, va delivery.ViewAnswer, at time.Time) bool {
|
||||
k, err := delivery.ParseAnyAskKey(askKey)
|
||||
if err != nil || k.Kind != delivery.AskKindHold {
|
||||
return false
|
||||
}
|
||||
a, err := delivery.DecodeAnswer(cur)
|
||||
if err != nil || a.Option == va.Option {
|
||||
return false
|
||||
}
|
||||
_, when, _ := strings.Cut(a.Warrant, "@")
|
||||
before, err := time.Parse(time.RFC3339Nano, when)
|
||||
if err != nil || !at.After(before) {
|
||||
t.logf("%s: the view's answer %s is not newer than the answer %s that stands (%s), and is not taken", askKey, va.Option, a.Option, when)
|
||||
return false
|
||||
}
|
||||
t.logf("%s: the operator changed the answer from %s to %s in the view", askKey, a.Option, va.Option)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -10,8 +11,9 @@ import (
|
||||
|
||||
const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
|
||||
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, as a fixture (in release 5 the assigner writes
|
||||
// none yet; the value is `{"v":1,"ask":<the router's ask>}`).
|
||||
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, shaped as the assigner composes it
|
||||
// (mesh-delivery-assigner internal/assign asks.go HoldAsk: two options of level acknowledge that bind no act, so the router
|
||||
// takes it; the value is `{"v":1,"ask":<the router's ask>}`).
|
||||
func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
||||
t.Helper()
|
||||
key, err := delivery.HoldAskKey("plex", fpHeld)
|
||||
@@ -20,7 +22,7 @@ func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
||||
}
|
||||
ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator,
|
||||
Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held",
|
||||
Options: []asks.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: asks.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: asks.Approve}}}
|
||||
Options: []asks.Option{{ID: "keep", Label: "Keep it held", Does: "keeps it held", Level: asks.Acknowledge}, {ID: "release", Label: "Release it", Does: "lets the move go on", Level: asks.Acknowledge}}}
|
||||
value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -117,8 +119,11 @@ func TestAViewAnswerOfAnOptionTheAskDoesNotOfferFillsNothing(t *testing.T) { //
|
||||
}
|
||||
}
|
||||
|
||||
// A view answer written before the answer that stands never replaces it (1906/N23: only a later one may, for a hold). The
|
||||
// view's time is fixed here, not the package's clock, which the integration tests set to the wall clock.
|
||||
func TestAViewAnswerNeverReplacesTheAnswerOfTheSameAsking(t *testing.T) { // 1297/B10
|
||||
b := newBed(t)
|
||||
b.t.Now = func() time.Time { return time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }
|
||||
assigner, view := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
@@ -174,3 +179,53 @@ func TestAStaleViewAnswerOfAnotherAskingFillsNothing(t *testing.T) { // 1297/B10
|
||||
t.Errorf("writes %d; log:\n%s", b.answers.writes, b.logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// timedView is the view's bucket with the time each answer was written.
|
||||
type timedView struct {
|
||||
memGate
|
||||
at map[string]time.Time
|
||||
}
|
||||
|
||||
func (v timedView) GetAt(key string) ([]byte, time.Time, bool, error) {
|
||||
b, ok := v.memGate[key]
|
||||
return b, v.at[key], ok, nil
|
||||
}
|
||||
|
||||
// 1906/N23-AC4: given a hold ask answered "keep" on the phone, when the operator answers "release" in the view later, then the
|
||||
// teller replaces the answer of that asking with "release", whose warrant names the asking and the view's time; a view answer
|
||||
// written before the phone's answer, the same option, or an ask of another kind replaces nothing.
|
||||
func TestALaterViewAnswerChangesAHoldsAnswer(t *testing.T) {
|
||||
setup := func(viewAt time.Time, option string) (*bed, string, asks.Ask) {
|
||||
b := newBed(t)
|
||||
assigner := memGate{}
|
||||
view := timedView{memGate: memGate{}, at: map[string]time.Time{}}
|
||||
b.t.Assigner, b.t.View = assigner, view
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + now.Add(time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator, via telegram, chose keep"})
|
||||
b.answers.written[key], b.answers.revs[key] = kept, 3
|
||||
view.memGate[key+".view"] = viewAnswer(t, option, "it was a flake")
|
||||
view.at[key+".view"] = viewAt
|
||||
b.t.OnViewAnswer(key + ".view")
|
||||
return b, key, ask
|
||||
}
|
||||
later := now.Add(2 * time.Hour)
|
||||
b, key, ask := setup(later, "release")
|
||||
a, err := delivery.DecodeAnswer(b.answers.written[key])
|
||||
if err != nil || b.answers.writes != 1 || a.Option != "release" || a.Warrant != ask.ID+"@"+later.UTC().Format(time.RFC3339Nano) {
|
||||
t.Fatalf("writes %d, answer %+v, %v; log:\n%s", b.answers.writes, a, err, b.logs.String())
|
||||
}
|
||||
for name, b := range map[string]*bed{
|
||||
"older": func() *bed { b, _, _ := setup(now, "release"); return b }(),
|
||||
"same": func() *bed { b, _, _ := setup(later, "keep"); return b }(),
|
||||
} {
|
||||
if b.answers.writes != 0 {
|
||||
t.Errorf("%s: the answer was replaced", name)
|
||||
}
|
||||
}
|
||||
// A restore ask answered stays answered: only a hold's answer may be changed.
|
||||
bb := newBed(t)
|
||||
if bb.t.changesMind("restore.ace.plex.2222222222222222222222222222222222222222222222222222222222222222", b.answers.written[key], delivery.ViewAnswer{Option: "keep"}, later.Add(time.Hour)) {
|
||||
t.Error("a restore ask's answer was changed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
// Command mesh-delivery-cli is the operator's command for the delivery view (novox/hq ADR 0429 §3). It asks the
|
||||
// view's local socket for the rows and prints them; it holds no credential. `answer <ask key> <option> [words]` asks the view
|
||||
// to write the operator's answer to an ask that needs no proof of who answered (the view refuses the merge's Approve).
|
||||
// to write the operator's answer to an ask that needs no proof of who answered (the view refuses the merge's Approve). `holds`
|
||||
// lists every fingerprint the assigner holds with its node, cause and answer; `hold-release <module> <fingerprint> [words]` and
|
||||
// `hold-keep <module> <fingerprint> [words]` answer its hold ask, again when the operator changes their mind (issue 1906).
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -10,6 +12,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
)
|
||||
@@ -31,9 +34,26 @@ func run(args []string, socket string, out, errOut io.Writer) int {
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() > 0 && fs.Arg(0) == "holds" {
|
||||
return holds(socket, *asJSON, out, errOut)
|
||||
}
|
||||
if fs.NArg() > 0 && (fs.Arg(0) == "hold-release" || fs.Arg(0) == "hold-keep") {
|
||||
if fs.NArg() < 3 {
|
||||
fmt.Fprintf(errOut, "mesh-delivery-cli %s <module> <fingerprint> [words]\n", fs.Arg(0))
|
||||
return 2
|
||||
}
|
||||
option := strings.TrimPrefix(fs.Arg(0), "hold-")
|
||||
r, err := view.AnswerHoldOn(socket, fs.Arg(1), fs.Arg(2), option, strings.Join(fs.Args()[3:], " "))
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintf(out, "answered the hold of %s at %s with %s; the teller fills the ask from it and the assigner reads it\n", r.Module, r.FP, option)
|
||||
return 0
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
if fs.Arg(0) != "answer" || fs.NArg() < 3 {
|
||||
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]")
|
||||
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]; holds; hold-release|hold-keep <module> <fingerprint> [words]")
|
||||
return 2
|
||||
}
|
||||
key, option := fs.Arg(1), fs.Arg(2)
|
||||
@@ -61,3 +81,53 @@ func run(args []string, socket string, out, errOut io.Writer) int {
|
||||
view.Table(out, s)
|
||||
return 0
|
||||
}
|
||||
|
||||
// holds prints every hold: module, fingerprint, node, since, answer, words.
|
||||
func holds(socket string, asJSON bool, out, errOut io.Writer) int {
|
||||
rows, err := view.AskHolds(socket)
|
||||
if err != nil {
|
||||
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
||||
return 1
|
||||
}
|
||||
if asJSON {
|
||||
enc := json.NewEncoder(out)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(rows); err != nil {
|
||||
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
fmt.Fprintln(out, "the assigner holds nothing")
|
||||
return 0
|
||||
}
|
||||
tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "MODULE\tFINGERPRINT\tNODE\tSINCE\tANSWER\tWORDS")
|
||||
for _, r := range rows {
|
||||
answer := r.Answer
|
||||
switch {
|
||||
case r.AskKey == "":
|
||||
answer = "no ask"
|
||||
case r.Pending != "":
|
||||
answer = r.Pending + " (from the view, the teller fills it)"
|
||||
case answer == "":
|
||||
answer = "not answered"
|
||||
}
|
||||
fp := strings.TrimPrefix(r.FP, "src2:")
|
||||
if len(fp) > 12 {
|
||||
fp = fp[:12]
|
||||
}
|
||||
since := ""
|
||||
if !r.Since.IsZero() {
|
||||
since = r.Since.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
words := r.Words
|
||||
if r.Error != "" {
|
||||
words = "unreadable: " + r.Error
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", r.Module, fp, r.Node, since, answer, words)
|
||||
}
|
||||
tw.Flush()
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -240,3 +240,46 @@ func TestTheAnswerCommandAnswersAnAskNeedingNoProof(t *testing.T) { // 1297/B9b
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/N23-AC3: `holds` lists the holds with node, cause and answer; `hold-release <module> <fingerprint>` and `hold-keep` answer
|
||||
// the hold ask through the view, also one answered already.
|
||||
func TestTheHoldCommandsListAndAnswerHolds(t *testing.T) {
|
||||
fp := "src2:04efd1a1aa" + strings.Repeat("2", 54)
|
||||
key, err := delivery.HoldAskKey("systemd-resolved", fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hold := asks.Ask{ID: delivery.AskID(key), Headline: "systemd-resolved is held", Explanation: "ace failed the check.", Who: asks.Operator, Expires: time.Now().Add(time.Hour), OnExpiry: "stays held",
|
||||
Options: []asks.Option{{ID: "keep", Label: "Keep", Does: "stays", Level: asks.Acknowledge}, {ID: "release", Label: "Release", Does: "goes on", Level: asks.Acknowledge}}}
|
||||
value, _ := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: hold})
|
||||
hkey, _ := delivery.HeldKey("systemd-resolved", fp)
|
||||
held, _ := delivery.EncodeHeld(delivery.Held{Node: "ace", Words: "health failed, phase running: dns timeout"})
|
||||
ans, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: hold.ID + "@2026-10-25T04:00:00Z", Words: "the operator chose keep"})
|
||||
m, w := view.New(), &putter{puts: map[string][]byte{}}
|
||||
m.SetWriter(w)
|
||||
m.Apply(view.Entry{Bucket: delivery.AssignerAsksBucket, Key: key, Value: value})
|
||||
m.Apply(view.Entry{Bucket: delivery.AssignerWantsBucket, Key: hkey, Value: held})
|
||||
m.Apply(view.Entry{Bucket: delivery.AnswersBucket, Key: key, Value: ans})
|
||||
socket := serveWith(t, m)
|
||||
|
||||
var out, errOut bytes.Buffer
|
||||
if code := run([]string{"holds"}, socket, &out, &errOut); code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||
}
|
||||
for _, want := range []string{"systemd-resolved", "04efd1a1aa22", "ace", "keep", "dns timeout"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("holds printed %q, missing %q", out.String(), want)
|
||||
}
|
||||
}
|
||||
out.Reset()
|
||||
if code := run([]string{"hold-release", "systemd-resolved", "04efd1a1", "a", "flake"}, socket, &out, &errOut); code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||
}
|
||||
if a, err := delivery.DecodeViewAnswer(w.puts[key+".view"]); err != nil || a.Option != "release" || a.Asking != hold.ID || a.Words != "a flake" {
|
||||
t.Errorf("the view's answer = %+v, %v", a, err)
|
||||
}
|
||||
errOut.Reset()
|
||||
if code := run([]string{"hold-keep", "systemd-resolved"}, socket, &out, &errOut); code != 2 {
|
||||
t.Errorf("hold-keep without a fingerprint: exit %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"mesh-delivery-view/internal/view"
|
||||
@@ -82,9 +83,50 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
commit, _ := args["commit"].(string)
|
||||
return r.BuilderBuild(module, fp2, commit)
|
||||
}},
|
||||
{Name: view.WantsReadTool,
|
||||
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
|
||||
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Wants(node, module)
|
||||
}},
|
||||
{Name: view.WantReadTool,
|
||||
Description: "One want of the assigner as stored: what the node is to run of the module (fingerprint, the newest it is held back from and the rule, removed, words). Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"node", "module"}, "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Want(node, module)
|
||||
}},
|
||||
{Name: view.DifferencesReadTool,
|
||||
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
|
||||
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Differences(node, module)
|
||||
}},
|
||||
{Name: view.HavesReadTool,
|
||||
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
|
||||
"and <node>.node (the node's own facts). Narrow by node and module. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Haves(node, module)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
var nodeModule = map[string]any{
|
||||
"node": map[string]any{"type": "string", "description": "only this node's keys"},
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}
|
||||
|
||||
func nodeModuleOf(args map[string]any) (string, string) {
|
||||
node, _ := args["node"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
return strings.TrimSpace(node), strings.TrimSpace(module)
|
||||
}
|
||||
|
||||
func read(state stdio.KeptState) view.StateRead {
|
||||
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
e, err := state.Get(key)
|
||||
@@ -102,7 +144,8 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
}
|
||||
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))}
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
|
||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -264,3 +264,68 @@ func TestCandidateToolsAnswerTheStoredValue(t *testing.T) {
|
||||
t.Errorf("by module = %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D1 AC1: assigner_wants, assigner_want, assigner_differences and node_haves answer as stored, narrowed by node and module.
|
||||
func keptReader() view.Reader {
|
||||
st := func(f fake) view.StateRead { return f.reader().State }
|
||||
return view.Reader{
|
||||
WantsState: st(fake{
|
||||
"want.ace.zsh": `{"fp":"` + fp2A + `"}`,
|
||||
"want.novox.zsh": `{"fp":"` + fp2B + `"}`,
|
||||
"want.ace.netcheck": `{"fp":"` + fp2A + `","rule":3}`,
|
||||
"held." + keyB: `{"node":"ace","words":"failed"}`,
|
||||
}),
|
||||
DifferencesState: st(fake{"ace.zsh": `{"unexplained":true}`, "novox.zsh": `{"rule":3}`}),
|
||||
HavesState: st(fake{"ace.zsh": `{"fp":"x"}`, "ace.node": `{"facts":1}`, "novox.zsh": `{"fp":"y"}`}),
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
|
||||
r := keptReader()
|
||||
out, err := run(t, r, view.WantsReadTool, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(view.WantsAnswer)
|
||||
if len(a.Wants) != 3 || len(a.Held) != 1 || string(a.Wants["want.ace.zsh"]) != `{"fp":"`+fp2A+`"}` {
|
||||
t.Errorf("all = %+v", a)
|
||||
}
|
||||
out, _ = run(t, r, view.WantsReadTool, map[string]any{"node": "ace"})
|
||||
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
|
||||
t.Errorf("node ace = %v", got)
|
||||
}
|
||||
out, _ = run(t, r, view.WantsReadTool, map[string]any{"module": "zsh"})
|
||||
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
|
||||
t.Errorf("module zsh = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
|
||||
r := keptReader()
|
||||
out, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "zsh"})
|
||||
if err != nil || string(out.(json.RawMessage)) != `{"fp":"`+fp2A+`"}` {
|
||||
t.Errorf("want = %v %v", out, err)
|
||||
}
|
||||
if _, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "gone"}); err == nil || !strings.Contains(err.Error(), "no want") {
|
||||
t.Errorf("a missing want: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
|
||||
r := keptReader()
|
||||
out, err := run(t, r, view.DifferencesReadTool, map[string]any{"node": "novox"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := out.(view.KeptAnswer).Keys; len(got) != 1 || string(got["novox.zsh"]) != `{"rule":3}` {
|
||||
t.Errorf("differences = %v", got)
|
||||
}
|
||||
out, _ = run(t, r, view.HavesReadTool, map[string]any{"module": "zsh"})
|
||||
if got := out.(view.KeptAnswer).Keys; len(got) != 2 {
|
||||
t.Errorf("haves of zsh = %v", got)
|
||||
}
|
||||
out, _ = run(t, r, view.HavesReadTool, map[string]any{"node": "ace"})
|
||||
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
|
||||
t.Errorf("haves of ace (the node's own key too) = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -433,17 +433,17 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerDifferencesBucket, History: delivery.AssignerDifferencesHistory})
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerWantsBucket, History: delivery.AssignerWantsHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
const fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
for key, d := range map[string]delivery.Difference{
|
||||
"ace.plex": {Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"},
|
||||
"g14.plex": {Want: fpOld, Sent: fpNew, Unexplained: true},
|
||||
for key, d := range map[string]delivery.Want{
|
||||
"want.ace.plex": {FP: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)},
|
||||
"want.g14.plex": {FP: fpOld, Composed: "sha256:" + strings.Repeat("c", 64), Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"},
|
||||
} {
|
||||
v, err := delivery.EncodeDifference(d)
|
||||
v, err := delivery.EncodeWant(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -465,13 +465,13 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
|
||||
t.Fatalf("%s; last: %+v", what, last)
|
||||
}
|
||||
wait(func(s view.Snapshot) bool {
|
||||
d := s.Differences
|
||||
return len(d) == 2 && d[0].Node == "ace" && d[0].Rule == delivery.RuleCanary && d[1].Node == "g14" && d[1].Unexplained
|
||||
}, "the two differences never showed with their rules")
|
||||
if err := kv.Delete(ctx, "g14.plex"); err != nil {
|
||||
d := s.Wants
|
||||
return len(d) == 2 && d[0].Node == "ace" && d[0].State == view.WantPending && d[1].Node == "g14" && d[1].State == view.WantHeldBack && d[1].Rule == delivery.RuleCanary
|
||||
}, "the two wants never showed with where they stand")
|
||||
if err := kv.Delete(ctx, "want.g14.plex"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait(func(s view.Snapshot) bool { return len(s.Differences) == 1 }, "the difference taken away never left the list")
|
||||
wait(func(s view.Snapshot) bool { return len(s.Wants) == 1 }, "the want taken away never left the list")
|
||||
}
|
||||
|
||||
// TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof starts the program whole, with the assigner's asks, the view's
|
||||
@@ -696,3 +696,230 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC
|
||||
}
|
||||
t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog)
|
||||
}
|
||||
|
||||
// 1906/D9, whole on the bus: when the wants and the haves make the source revision delivered, the view says it once, as its event
|
||||
// `delivered` with the revision, the group and the heads, which the issue tracker derives "fix delivered" from.
|
||||
func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) {
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, name := range []string{delivery.SourceBucket, delivery.ModulesBucket, delivery.AssignerWantsBucket, delivery.HaveBucket} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[name] = kv
|
||||
}
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
fp := "src2:" + strings.Repeat("2", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
good, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}})
|
||||
for bucket, kv := range map[string][]byte{delivery.ModulesBucket: mod} {
|
||||
if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := kvs[delivery.SourceBucket].Put(ctx, delivery.SourceMainKey, src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if info, err := events.Info(ctx); err != nil || info.State.Msgs != 0 {
|
||||
t.Fatalf("an event was said before the revision was delivered: %v %+v", err, info.State)
|
||||
}
|
||||
if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var msg *jetstream.RawStreamMsg
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && msg == nil; time.Sleep(100 * time.Millisecond) {
|
||||
msg, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+DeliveredEventType)
|
||||
}
|
||||
if msg == nil {
|
||||
t.Fatal("the delivery of the source revision was never said")
|
||||
}
|
||||
var e view.DeliveredEvent
|
||||
if err := json.Unmarshal(msg.Data, &e); err != nil || e.Revision != rev || e.Group != "feat/x" || e.Heads["novox/web"] != strings.Repeat("1", 40) {
|
||||
t.Errorf("the event = %+v, %v; want revision %d, group feat/x and the heads", e, err, rev)
|
||||
}
|
||||
if msg.Header.Get("x-source") != "mesh-delivery-view" || msg.Header.Get("x-event-id") == "" {
|
||||
t.Errorf("the headers = %v", msg.Header)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and
|
||||
// healthy `settled`, a have held or failed `stuck`, each once with the body the messenger reads and an event id per type, node, module
|
||||
// and composition.
|
||||
func TestProcessWholeSaysModulesInMotion(t *testing.T) {
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, name := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[name] = kv
|
||||
}
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
time.Sleep(2 * time.Second) // the view has read what stood: nothing
|
||||
|
||||
fpOld, fpNew := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64)
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
put := func(bucket, key string, v []byte, err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kvs[bucket].Put(ctx, key, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want := func(node, module string) {
|
||||
v, err := delivery.EncodeWant(delivery.Want{FP: fpNew, Composed: digest})
|
||||
put(delivery.AssignerWantsBucket, "want."+node+"."+module, v, err)
|
||||
}
|
||||
have := func(node, module, fp string, phase delivery.Phase, health delivery.Health, detail string) {
|
||||
v, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: phase, Health: health, Detail: detail})
|
||||
put(delivery.HaveBucket, node+"."+module, v, err)
|
||||
}
|
||||
have("ace", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "")
|
||||
want("ace", "web")
|
||||
have("g14", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "")
|
||||
want("g14", "web")
|
||||
|
||||
said := func() map[string]jetstream.RawStreamMsg {
|
||||
out := map[string]jetstream.RawStreamMsg{}
|
||||
for seq := uint64(1); ; seq++ {
|
||||
m, err := events.GetMsg(ctx, seq)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
out[m.Header.Get("x-event-id")] = *m
|
||||
}
|
||||
}
|
||||
waitFor := func(id string) jetstream.RawStreamMsg {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if m, ok := said()[id]; ok {
|
||||
return m
|
||||
}
|
||||
}
|
||||
t.Fatalf("never said %s; said %v", id, func() []string {
|
||||
var ids []string
|
||||
for k := range said() {
|
||||
ids = append(ids, k)
|
||||
}
|
||||
return ids
|
||||
}())
|
||||
return jetstream.RawStreamMsg{}
|
||||
}
|
||||
hex := strings.Repeat("c", 64)
|
||||
mv := waitFor("moving.ace.web." + hex)
|
||||
var body view.MotionBody
|
||||
if err := json.Unmarshal(mv.Data, &body); err != nil || body.Node != "ace" || body.Module != "web" || body.FromFP != fpOld || body.ToFP != fpNew || body.Composed != digest || body.BoundSeconds != view.MotionBoundSeconds || body.Since.IsZero() {
|
||||
t.Errorf("moving = %+v, %v", body, err)
|
||||
}
|
||||
if mv.Subject != "mesh.mod.mesh-delivery-view.event.moving" {
|
||||
t.Errorf("subject = %s", mv.Subject)
|
||||
}
|
||||
have("ace", "web", fpNew, delivery.PhaseRunning, delivery.HealthOK, "")
|
||||
st := waitFor("settled.ace.web." + hex)
|
||||
if err := json.Unmarshal(st.Data, &body); err != nil || body.Outcome != "running" {
|
||||
t.Errorf("settled = %+v, %v", body, err)
|
||||
}
|
||||
have("g14", "web", fpOld, delivery.PhaseHeld, delivery.HealthWaits, "no space left")
|
||||
stk := waitFor("stuck.g14.web." + hex)
|
||||
if err := json.Unmarshal(stk.Data, &body); err != nil || body.Outcome != "stopped" || body.Words != "no space left" {
|
||||
t.Errorf("stuck = %+v, %v", body, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
count := map[string]int{}
|
||||
for id := range said() {
|
||||
count[strings.SplitN(id, ".", 2)[0]]++
|
||||
}
|
||||
if count["moving"] != 2 || count["settled"] != 1 || count["stuck"] != 1 {
|
||||
t.Errorf("events said by type = %v, want moving 2, settled 1, stuck 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,8 +138,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
model.SetWriter(kvWriter{ctx: ctx, js: js})
|
||||
model.SetMotionEvent(func(e view.MotionEvent) {
|
||||
if err := emitEvent(ctx, js, cred, e.Type, e.Body, e.ID); err != nil {
|
||||
logger.Printf("the %s of %s on %s could not be said: %v", e.Type, e.Body.Module, e.Body.Node, err)
|
||||
}
|
||||
})
|
||||
model.SetDeliveredEvent(func(e view.DeliveredEvent) {
|
||||
if err := emitEvent(ctx, js, cred, DeliveredEventType, e, fmt.Sprintf("delivered.%d", e.Revision)); err != nil {
|
||||
logger.Printf("the delivery of source revision %d could not be said: %v", e.Revision, err)
|
||||
}
|
||||
})
|
||||
model.SetBacklog(view.BacklogOver(js, 5*time.Second))
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} {
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} {
|
||||
go follow(ctx, js, bucket, model, logger)
|
||||
}
|
||||
select {
|
||||
@@ -204,10 +214,36 @@ func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, model
|
||||
return errors.New("the watch ended")
|
||||
}
|
||||
if e == nil { // the last values have all been delivered
|
||||
model.Initial(bucket)
|
||||
continue
|
||||
}
|
||||
model.Apply(view.Entry{Bucket: bucket, Key: e.Key(), Value: e.Value(), At: e.Created(),
|
||||
Deleted: e.Operation() != jetstream.KeyValuePut})
|
||||
Deleted: e.Operation() != jetstream.KeyValuePut, Revision: e.Revision()})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeliveredEventType is the event the view says when a source revision is delivered: mesh.mod.mesh-delivery-view.event.delivered
|
||||
// (issue 1906 D9, ADR 0539 §17). Its body is view.DeliveredEvent.
|
||||
const DeliveredEventType = "delivered"
|
||||
|
||||
// emitEvent publishes an event of the view's own namespace, mesh.mod.<module>.event.<type>, with the SDK's envelope headers
|
||||
// (as mesh-sdk's process.Conn.Emit does) and the id as the bus's own dedup id, so the same revision said twice in the stream's
|
||||
// duplicate window is kept once.
|
||||
func emitEvent(ctx context.Context, js jetstream.JetStream, cred credential, eventType string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + eventType)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// diffEntry is a key of the assigner's differences as the assigner writes it (mesh-sdk go/delivery EncodeDifference).
|
||||
func differenceE(t *testing.T, node, module string, d delivery.Difference) Entry {
|
||||
t.Helper()
|
||||
key, err := delivery.DifferenceKey(node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := delivery.EncodeDifference(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AssignerDifferencesBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func TestEveryDifferenceIsListedWithItsRule(t *testing.T) { // 1297/B8 AC
|
||||
m := New()
|
||||
fill(m,
|
||||
differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}),
|
||||
differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}),
|
||||
differenceE(t, "ace", "old", delivery.Difference{WantRemoved: true, Sent: fpOld, Rule: delivery.RuleNotHeld}),
|
||||
)
|
||||
got := m.Snapshot().Differences
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("differences = %+v, want all three", got)
|
||||
}
|
||||
// ordered by node then module
|
||||
if got[0].Node != "ace" || got[0].Module != "old" || got[1].Module != "plex" || got[2].Node != "g14" {
|
||||
t.Errorf("order = %+v", got)
|
||||
}
|
||||
if r := got[1]; r.Want != fpOld || r.Sent != fpNew || r.Rule != delivery.RuleCanary || r.Unexplained || r.Words != "ace is not the canary" {
|
||||
t.Errorf("row = %+v", r)
|
||||
}
|
||||
if !got[0].WantRemoved || got[0].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("removed row = %+v", got[0])
|
||||
}
|
||||
if !got[2].Unexplained || got[2].Rule != 0 {
|
||||
t.Errorf("unexplained row = %+v", got[2])
|
||||
}
|
||||
// a difference taken away (the want and the send agree again) leaves the list
|
||||
fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "g14.plex", Deleted: true})
|
||||
if got := m.Snapshot().Differences; len(got) != 2 {
|
||||
t.Errorf("after the delete = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableDifferenceIsNamedAndNotDropped(t *testing.T) { // never dropped silently
|
||||
m := New()
|
||||
fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "ace.plex", Value: []byte(`{"v":99}`), At: t0})
|
||||
s := m.Snapshot()
|
||||
if len(s.Differences) != 0 || len(s.Unreadable) != 1 || s.Unreadable[0].Bucket != delivery.AssignerDifferencesBucket || s.Unreadable[0].Key != "ace.plex" {
|
||||
t.Errorf("snapshot = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableAndPageShowTheWantsAgainstTheSends(t *testing.T) { // 1297/B8 AC
|
||||
m := New()
|
||||
fill(m,
|
||||
differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}),
|
||||
differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}),
|
||||
)
|
||||
var out strings.Builder
|
||||
Table(&out, m.Snapshot())
|
||||
for _, w := range []string{"WANTS AGAINST SENDS", "ace", "plex", fpOld, fpNew, "rule 3", "ace is not the canary", "unexplained"} {
|
||||
if !strings.Contains(out.String(), w) {
|
||||
t.Errorf("table lacks %q:\n%s", w, out.String())
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/", "/wants"} {
|
||||
rec := httptest.NewRecorder()
|
||||
Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
body := rec.Body.String()
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%s = %d", path, rec.Code)
|
||||
}
|
||||
for _, w := range []string{"WANTS AGAINST SENDS", "rule 3", "unexplained", "ace is not the canary"} {
|
||||
if !strings.Contains(body, w) {
|
||||
t.Errorf("%s lacks %q:\n%s", path, w, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
// With none, the page says so and does not hide the heading.
|
||||
rec := httptest.NewRecorder()
|
||||
Web(New()).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/wants", nil))
|
||||
if !strings.Contains(rec.Body.String(), "no difference") {
|
||||
t.Errorf("empty page = %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's holds, listed and answered from the operator's command (issue 1906 note 23). A hold ask answered once (on the
|
||||
// phone or in the view) ended with that answer, so nothing listed it any more and an answer "keep" could not be changed: the
|
||||
// operator kept systemd-resolved held and then could not release it, and a kept hold is never lifted by itself. Now the view
|
||||
// lists every hold with its node, its cause and its answer, and answers the hold ask that stands, again if the operator changes
|
||||
// their mind, through the view's one write (`<ask key>.view`), which the teller takes into the ask's answer. Nothing else writes:
|
||||
// the answer still reaches the assigner only through the teller (mesh-delivery-teller viewanswers.go).
|
||||
|
||||
// The options of the hold ask (mesh-delivery-assigner internal/assign asks.go HoldAsk).
|
||||
const (
|
||||
HoldRelease = "release"
|
||||
HoldKeep = "keep"
|
||||
)
|
||||
|
||||
type heldEntry struct {
|
||||
module, fp string
|
||||
value delivery.Held
|
||||
err string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// applyHeld takes a change of a held key of the assigner's wants, and says whether the key was one.
|
||||
func (m *Model) applyHeld(e Entry) bool {
|
||||
module, fp, err := delivery.ParseHeldKey(e.Key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if e.Deleted {
|
||||
delete(m.holds, e.Key)
|
||||
return true
|
||||
}
|
||||
h, err := delivery.DecodeHeld(e.Value)
|
||||
m.holds[e.Key] = heldEntry{module: module, fp: fp, value: h, err: errText(err), at: e.At}
|
||||
return true
|
||||
}
|
||||
|
||||
// HoldRow is one fingerprint the assigner holds: the module, the fingerprint, the node whose health check failed, the hold's
|
||||
// words (its cause) and since when; the ask about it (its key and the asking's id, empty when the assigner asks none), and the
|
||||
// answer that stands for that asking (the option and its words, empty when none), with the view's own answer when it waits for
|
||||
// the teller.
|
||||
type HoldRow struct {
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Since time.Time `json:"since,omitempty"`
|
||||
AskKey string `json:"ask-key,omitempty"`
|
||||
Asking string `json:"asking,omitempty"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered string `json:"answered,omitempty"`
|
||||
Pending string `json:"pending,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Holds is every hold, ordered by module and fingerprint.
|
||||
func (m *Model) Holds() []HoldRow {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
out := []HoldRow{}
|
||||
for _, h := range m.holds {
|
||||
row := HoldRow{Module: h.module, FP: h.fp, Node: h.value.Node, Words: h.value.Words, Since: h.at, Error: h.err}
|
||||
if key, err := delivery.HoldAskKey(h.module, h.fp); err == nil {
|
||||
if a, ok := m.aasks[key]; ok && a.err == "" {
|
||||
row.AskKey, row.Asking = key, a.ask.ID
|
||||
if ans, ok := m.answerOf[key]; ok && m.answers[key] == a.ask.ID {
|
||||
row.Answer, row.Answered = ans.Option, ans.Words
|
||||
}
|
||||
if c := m.chosenOf(key, a.ask); c != "" && c != row.Answer {
|
||||
row.Pending = c
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Module != out[j].Module {
|
||||
return out[i].Module < out[j].Module
|
||||
}
|
||||
return out[i].FP < out[j].FP
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// AnswerHold writes the view's answer to the hold ask of module at the fingerprint fp: release or keep, with the operator's
|
||||
// words. fp is the whole fingerprint, or the start of its hex (as the rows print it), naming one hold of the module. Unlike
|
||||
// Answer it answers an ask answered already, so the operator can change their mind; an answer equal to the one that stands is
|
||||
// refused, as is a hold the assigner asks nothing about.
|
||||
func (m *Model) AnswerHold(module, fp, option, words string) (HoldRow, error) {
|
||||
option = strings.TrimSpace(option)
|
||||
if option != HoldRelease && option != HoldKeep {
|
||||
return HoldRow{}, fmt.Errorf("a hold is answered %s or %s, not %q", HoldRelease, HoldKeep, option)
|
||||
}
|
||||
var match []HoldRow
|
||||
want := strings.TrimPrefix(strings.TrimSpace(fp), delivery.SourcePrefix2)
|
||||
for _, r := range m.Holds() {
|
||||
if r.Module == module && want != "" && strings.HasPrefix(strings.TrimPrefix(r.FP, delivery.SourcePrefix2), want) {
|
||||
match = append(match, r)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(match) == 0:
|
||||
return HoldRow{}, fmt.Errorf("the assigner holds no fingerprint %q of %s (mesh-delivery-cli holds lists them)", fp, module)
|
||||
case len(match) > 1:
|
||||
return HoldRow{}, fmt.Errorf("%q names %d holds of %s: give more of the fingerprint", fp, len(match), module)
|
||||
}
|
||||
r := match[0]
|
||||
if r.AskKey == "" {
|
||||
return r, fmt.Errorf("the assigner asks nothing about the hold of %s at %s, so there is nothing to answer", module, r.FP)
|
||||
}
|
||||
if r.Answer == option && r.Pending == "" {
|
||||
return r, fmt.Errorf("the hold of %s at %s is answered %s already", module, r.FP, option)
|
||||
}
|
||||
m.mu.Lock()
|
||||
a, w := m.aasks[r.AskKey], m.writer
|
||||
m.mu.Unlock()
|
||||
if !slices.ContainsFunc(a.ask.Options, func(o asks.Option) bool { return o.ID == option }) {
|
||||
return r, fmt.Errorf("the hold ask %s offers no option %q", r.AskKey, option)
|
||||
}
|
||||
if w == nil {
|
||||
return r, fmt.Errorf("the view cannot write its answers here")
|
||||
}
|
||||
key, err := delivery.ViewAnswerKey(r.AskKey)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
body, err := delivery.EncodeViewAnswer(delivery.ViewAnswer{Asking: a.ask.ID, Option: option, Words: strings.TrimSpace(words)})
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
if err := w.Put(key, body); err != nil {
|
||||
return r, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// askOn sends one ask to the socket at path and decodes the reply into out; the view's refusal is the error.
|
||||
func askOn(path string, req ask, out any) error {
|
||||
c, err := net.DialTimeout("unix", path, 5*time.Second)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the view at %s: %w", path, err)
|
||||
}
|
||||
defer c.Close()
|
||||
c.SetDeadline(time.Now().Add(10 * time.Second))
|
||||
raw, _ := json.Marshal(req)
|
||||
if _, err := c.Write(append(raw, '\n')); err != nil {
|
||||
return err
|
||||
}
|
||||
var reply json.RawMessage
|
||||
if err := json.NewDecoder(c).Decode(&reply); err != nil {
|
||||
return fmt.Errorf("the view's reply cannot be read: %w", err)
|
||||
}
|
||||
var refusal struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(reply, &refusal) == nil && refusal.Error != "" {
|
||||
return errors.New(refusal.Error)
|
||||
}
|
||||
return json.Unmarshal(reply, out)
|
||||
}
|
||||
|
||||
// AskHolds asks the socket at path for every hold.
|
||||
func AskHolds(path string) ([]HoldRow, error) {
|
||||
var out []HoldRow
|
||||
err := askOn(path, ask{Ask: "holds"}, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// AnswerHoldOn asks the socket at path to answer the hold ask of module at fp with option.
|
||||
func AnswerHoldOn(path, module, fp, option, words string) (HoldRow, error) {
|
||||
var out HoldRow
|
||||
err := askOn(path, ask{Ask: "hold", Module: module, FP: fp, Option: option, Words: words}, &out)
|
||||
return out, err
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// heldE is the assigner's hold of module at fp, as it writes it in its wants.
|
||||
func heldE(t *testing.T, module, fp, node, words string) Entry {
|
||||
t.Helper()
|
||||
key, err := delivery.HeldKey(module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := delivery.EncodeHeld(delivery.Held{Node: node, Words: words})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AssignerWantsBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func answered(t *testing.T, key, id, option string) Entry {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: id + "@2026-10-25T04:00:00Z", Words: "the operator, via telegram, chose " + option})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AnswersBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
// 1906/N23-AC1: given holds in the assigner's wants, the view lists each with its node, its cause in words, its hold ask and
|
||||
// the answer that stands for that asking; a hold without an ask says so, and a held key is never taken for a want.
|
||||
func TestTheViewListsEveryHoldWithItsNodeCauseAndAnswer(t *testing.T) {
|
||||
m := New()
|
||||
e, a := holdAsk(t)
|
||||
fill(m, e, heldE(t, "plex", fpNew, "ace", "health failed, phase running: connection refused"), answered(t, e.Key, a.ID, "keep"),
|
||||
heldE(t, "web", fpNew, "g14", "health failed"))
|
||||
got := m.Holds()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("holds = %+v", got)
|
||||
}
|
||||
p := got[0]
|
||||
if p.Module != "plex" || p.FP != fpNew || p.Node != "ace" || !strings.Contains(p.Words, "connection refused") || p.AskKey != e.Key || p.Asking != a.ID || p.Answer != "keep" {
|
||||
t.Errorf("plex = %+v", p)
|
||||
}
|
||||
if w := got[1]; w.Module != "web" || w.Node != "g14" || w.AskKey != "" || w.Answer != "" {
|
||||
t.Errorf("web = %+v", w)
|
||||
}
|
||||
if len(m.Snapshot().Wants) != 0 {
|
||||
t.Errorf("a hold was taken for a want: %+v", m.Snapshot().Wants)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/N23-AC2: given a hold ask answered "keep" (the ask ended with it), when the operator releases the hold, then the view
|
||||
// writes its answer "release" to that asking under the ask's view key, though the ask was answered; the fingerprint may be the
|
||||
// start of its hex. The same answer again, an unknown hold and another option are refused, and write nothing.
|
||||
func TestAKeptHoldCanBeReleased(t *testing.T) {
|
||||
m, w := New(), &memWriter{}
|
||||
m.SetWriter(w)
|
||||
e, a := holdAsk(t)
|
||||
fill(m, e, heldE(t, "plex", fpNew, "ace", "health failed"), answered(t, e.Key, a.ID, "keep"))
|
||||
short := strings.TrimPrefix(fpNew, delivery.SourcePrefix2)[:8]
|
||||
if _, err := m.AnswerHold("plex", short, HoldRelease, "it was a flake"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
va, err := delivery.DecodeViewAnswer(w.puts[e.Key+".view"])
|
||||
if err != nil || va.Asking != a.ID || va.Option != HoldRelease || va.Words != "it was a flake" {
|
||||
t.Fatalf("the view's answer = %+v, %v", va, err)
|
||||
}
|
||||
w.puts = nil
|
||||
fill(m, answered(t, e.Key, a.ID, HoldRelease))
|
||||
for name, try := range map[string]func() error{
|
||||
"again": func() error { _, err := m.AnswerHold("plex", fpNew, HoldRelease, ""); return err },
|
||||
"unknown": func() error { _, err := m.AnswerHold("plex", "ffff", HoldKeep, ""); return err },
|
||||
"option": func() error { _, err := m.AnswerHold("plex", fpNew, "restore", ""); return err },
|
||||
"module": func() error { _, err := m.AnswerHold("web", fpNew, HoldKeep, ""); return err },
|
||||
} {
|
||||
if err := try(); err == nil {
|
||||
t.Errorf("%s: not refused", name)
|
||||
}
|
||||
}
|
||||
if len(w.puts) != 0 {
|
||||
t.Errorf("a refused answer wrote %v", w.puts)
|
||||
}
|
||||
// And kept again: the operator may change their mind both ways.
|
||||
if _, err := m.AnswerHold("plex", fpNew, HoldKeep, ""); err != nil {
|
||||
t.Errorf("keep after release: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,10 @@ type Reader struct {
|
||||
State StateRead // the indexer's modules
|
||||
Builds StateRead // the builder's builds
|
||||
|
||||
CandidatesState StateRead // the indexer's candidates
|
||||
CandidatesState StateRead // the indexer's candidates
|
||||
WantsState StateRead // the assigner's wants and holds
|
||||
DifferencesState StateRead // the assigner's differences
|
||||
HavesState StateRead // the node-engine's haves
|
||||
}
|
||||
|
||||
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
||||
@@ -328,3 +331,132 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
|
||||
}
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// The read tools of the assigner's wants and differences and of the node-engine's haves (issue 1906, D1): the keys of those
|
||||
// buckets answered as stored, so what the assigner wants, what it found different and what each node has can be read live.
|
||||
// They write nothing.
|
||||
const (
|
||||
WantsReadTool = "assigner_wants"
|
||||
WantReadTool = "assigner_want"
|
||||
DifferencesReadTool = "assigner_differences"
|
||||
HavesReadTool = "node_haves"
|
||||
)
|
||||
|
||||
// WantsAnswer is the answer of assigner_wants: the wants by key (`want.<node>.<module>`, a node's own `want.<node>.node`
|
||||
// among them) and the holds by key
|
||||
// (`held.<module>.<hex>`), each as stored, and the keys that are neither.
|
||||
type WantsAnswer struct {
|
||||
Wants map[string]json.RawMessage `json:"wants"`
|
||||
Held map[string]json.RawMessage `json:"held"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
}
|
||||
|
||||
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
|
||||
// are no `<node>.<module>` key.
|
||||
type KeptAnswer struct {
|
||||
Keys map[string]json.RawMessage `json:"keys"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
}
|
||||
|
||||
func (s StateRead) all(what string) (map[string]json.RawMessage, error) {
|
||||
if s.Keys == nil || s.Get == nil {
|
||||
return nil, fmt.Errorf("%s cannot be read here", what)
|
||||
}
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := map[string]json.RawMessage{}
|
||||
for _, k := range keys {
|
||||
raw, err := s.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
|
||||
// matches).
|
||||
func (r Reader) Wants(node, module string) (WantsAnswer, error) {
|
||||
all, err := r.WantsState.all("the assigner's wants")
|
||||
if err != nil {
|
||||
return WantsAnswer{}, err
|
||||
}
|
||||
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
|
||||
for k, raw := range all {
|
||||
if n, m, err := delivery.ParseWantKey(k); err == nil {
|
||||
if (node == "" || n == node) && (module == "" || m == module) {
|
||||
out.Wants[k] = raw
|
||||
}
|
||||
continue
|
||||
}
|
||||
// A node's own want, want.<node>.node (ADR 0539 clause 6): listed with the wants, under the module name "node".
|
||||
if n, err := delivery.ParseWantNodeKey(k); err == nil {
|
||||
if (node == "" || n == node) && (module == "" || module == "node") {
|
||||
out.Wants[k] = raw
|
||||
}
|
||||
continue
|
||||
}
|
||||
if m, _, err := delivery.ParseHeldKey(k); err == nil {
|
||||
if node == "" && (module == "" || m == module) {
|
||||
out.Held[k] = raw
|
||||
}
|
||||
continue
|
||||
}
|
||||
out.Unreadable = append(out.Unreadable, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Want is the stored want of module on node.
|
||||
func (r Reader) Want(node, module string) (json.RawMessage, error) {
|
||||
key, err := delivery.WantKey(strings.TrimSpace(node), strings.TrimSpace(module))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.WantsState.Get == nil {
|
||||
return nil, fmt.Errorf("the assigner's wants cannot be read here")
|
||||
}
|
||||
raw, err := r.WantsState.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("the assigner has no want %s", key)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
|
||||
all, err := s.all(what)
|
||||
if err != nil {
|
||||
return KeptAnswer{}, err
|
||||
}
|
||||
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
|
||||
for k, raw := range all {
|
||||
n, m, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil {
|
||||
out.Unreadable = append(out.Unreadable, k)
|
||||
continue
|
||||
}
|
||||
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
|
||||
out.Keys[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Differences is the assigner's differences as stored, narrowed by node and module.
|
||||
func (r Reader) Differences(node, module string) (KeptAnswer, error) {
|
||||
return r.kept(r.DifferencesState, "the assigner's differences", node, module)
|
||||
}
|
||||
|
||||
// Haves is the node-engine's haves (and each node's own key) as stored, narrowed by node and module.
|
||||
func (r Reader) Haves(node, module string) (KeptAnswer, error) {
|
||||
return r.kept(r.HavesState, "the haves", node, module)
|
||||
}
|
||||
|
||||
@@ -46,6 +46,8 @@ type ask struct {
|
||||
Key string `json:"key,omitempty"`
|
||||
Option string `json:"option,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
}
|
||||
|
||||
// ServeSocket answers {"ask":"rows"} on l, line-delimited JSON, until ctx ends, and {"ask":"answer","key":…,"option":…,"words":…}
|
||||
@@ -72,8 +74,21 @@ func answer(c net.Conn, m *Model) {
|
||||
return
|
||||
}
|
||||
var a ask
|
||||
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer") {
|
||||
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"} and {"ask":"answer"} and nothing else`})
|
||||
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer" && a.Ask != "holds" && a.Ask != "hold") {
|
||||
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"}, {"ask":"answer"}, {"ask":"holds"} and {"ask":"hold"} and nothing else`})
|
||||
return
|
||||
}
|
||||
switch a.Ask {
|
||||
case "holds":
|
||||
json.NewEncoder(c).Encode(m.Holds())
|
||||
return
|
||||
case "hold":
|
||||
r, err := m.AnswerHold(a.Module, a.FP, a.Option, a.Words)
|
||||
if err != nil {
|
||||
json.NewEncoder(c).Encode(map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
json.NewEncoder(c).Encode(r)
|
||||
return
|
||||
}
|
||||
if a.Ask == "answer" {
|
||||
@@ -209,33 +224,42 @@ func askText(s Snapshot) [][]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// differenceText is one line per difference: the node, the module, what the assigner wants, what the controller sends, the rule
|
||||
// that explains it (or unexplained) and the assigner's words.
|
||||
func differenceText(s Snapshot) [][]string {
|
||||
// wantText is one line per want: the node, the module, what the assigner wants, what the node runs, where the two stand and the
|
||||
// assigner's words (the rule that holds a node back, when one does).
|
||||
func wantText(s Snapshot) [][]string {
|
||||
var out [][]string
|
||||
for _, d := range s.Differences {
|
||||
want, sent := d.Want, d.Sent
|
||||
switch {
|
||||
case d.WantRemoved:
|
||||
want = "removed"
|
||||
case want == "":
|
||||
for _, w := range s.Wants {
|
||||
want, have := w.Want, w.Have
|
||||
if want == "" {
|
||||
want = "none"
|
||||
}
|
||||
if sent == "" {
|
||||
sent = "none"
|
||||
if d.SentBuild != "" {
|
||||
sent = "build " + d.SentBuild
|
||||
}
|
||||
if have == "" {
|
||||
have = "none"
|
||||
}
|
||||
why := "unexplained"
|
||||
if !d.Unexplained {
|
||||
why = fmt.Sprintf("rule %d", d.Rule)
|
||||
words := w.Words
|
||||
if w.Rule != 0 {
|
||||
words = fmt.Sprintf("rule %d: %s", w.Rule, w.Words)
|
||||
}
|
||||
out = append(out, []string{d.Node, d.Module, want, sent, why, d.Words})
|
||||
judged := w.Judgement
|
||||
if judged == "" {
|
||||
judged = "none"
|
||||
} else if w.Judged != "" {
|
||||
words = strings.TrimSpace(words + " " + w.Judged)
|
||||
}
|
||||
out = append(out, []string{w.Node, w.Module, want, have, w.State, judged, words})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sourceText is the source revision's derived state in one line, then one line for each module on a node it waits for.
|
||||
func sourceText(s Snapshot) []string {
|
||||
if s.Source == nil {
|
||||
return nil
|
||||
}
|
||||
out := []string{fmt.Sprintf("source revision %d: %s", s.Source.Revision, s.Source.State)}
|
||||
return append(out, s.Source.Waiting...)
|
||||
}
|
||||
|
||||
// backlogText is one line per language and kind: its waiting asks, the asks held by a node, the oldest waiting ask's age.
|
||||
func backlogText(s Snapshot) [][]string {
|
||||
var out [][]string
|
||||
@@ -254,7 +278,7 @@ func backlogText(s Snapshot) [][]string {
|
||||
|
||||
var backlogHead = []string{"LANGUAGE", "KIND", "WAITING", "HELD", "OLDEST WAITING"}
|
||||
|
||||
var differenceHead = []string{"NODE", "MODULE", "WANT", "SENT", "RULE", "WORDS"}
|
||||
var wantHead = []string{"NODE", "MODULE", "WANT", "HAVE", "STATE", "MOVE JUDGED", "WORDS"}
|
||||
|
||||
var head = []string{"NODE", "MODULE", "ASSIGNED", "HAVE", "PHASE", "HEALTH", "SINCE", "DETAIL", "WAITS"}
|
||||
|
||||
@@ -303,10 +327,16 @@ func Table(w io.Writer, s Snapshot) {
|
||||
row(cells)
|
||||
}
|
||||
}
|
||||
if d := differenceText(s); len(d) > 0 {
|
||||
if src := sourceText(s); len(src) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"WANTS AGAINST SENDS"})
|
||||
row(differenceHead)
|
||||
for _, line := range src {
|
||||
row([]string{line})
|
||||
}
|
||||
}
|
||||
if d := wantText(s); len(d) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"WANTS AGAINST HAVES"})
|
||||
row(wantHead)
|
||||
for _, cells := range d {
|
||||
row(cells)
|
||||
}
|
||||
@@ -356,16 +386,16 @@ var page = template.Must(template.New("p").Parse(`<!doctype html><html><head><me
|
||||
{{if .Groups}}<h2>CANDIDATES</h2><table><tr><th>GROUP</th><th>PULL REQUEST</th><th>HEAD</th><th>MODULES</th></tr>{{range .Groups}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Asks}}<h2>ASKS</h2><table><tr><th>ASK</th><th>HEADLINE</th><th>WORDS</th><th>PULL REQUESTS</th><th>ANSWER</th></tr>{{range .Asks}}<tr>{{range .Cells}}<td>{{.}}</td>{{end}}<td>{{if .Options}}<form method="post" action="/answer"><input type="hidden" name="key" value="{{.Key}}"><input name="words" placeholder="words (optional)">{{range .Options}} <button name="option" value="{{.}}">{{.}}</button>{{end}}</form>{{end}}</td></tr>{{end}}</table>{{end}}
|
||||
{{if .Backlog}}<h2>BACKLOG OF BUILD ASKS</h2><table><tr>{{range .BacklogHead}}<th>{{.}}</th>{{end}}</tr>{{range .Backlog}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Differences}}<h2>WANTS AGAINST SENDS</h2><table><tr>{{range .DifferenceHead}}<th>{{.}}</th>{{end}}</tr>{{range .Differences}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Source}}<h2>SOURCE REVISION</h2><p>{{range .Source}}{{.}}<br>{{end}}</p>{{end}}{{if .Wants}}<h2>WANTS AGAINST HAVES</h2><table><tr>{{range .WantHead}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Fingerprints}}<h2>FINGERPRINTS</h2><table><tr><th>MODULE</th><th>STATE</th><th>INDEXER AGAINST BUILD</th></tr>{{range .Fingerprints}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Offline}}<h2>OFFLINE</h2><table>{{range .Offline}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Stuck}}<h2>STUCK</h2><table><tr><th>BUCKET</th><th>KEY</th><th>WAITED FOR</th><th>SINCE</th></tr>{{range .Stuck}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Unreadable}}<h2>UNREADABLE</h2><table>{{range .Unreadable}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}</body></html>`))
|
||||
|
||||
var wantsPage = template.Must(template.New("w").Parse(`<!doctype html><html><head><meta charset="utf-8"><meta http-equiv="refresh" content="5">
|
||||
<title>Wants against sends</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
|
||||
<body><p><a href="/">What each node runs</a></p><h1>WANTS AGAINST SENDS</h1>
|
||||
{{if .Differences}}<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>{{range .Differences}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
|
||||
<title>Wants against haves</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
|
||||
<body><p><a href="/">What each node runs</a></p><h1>WANTS AGAINST HAVES</h1>
|
||||
{{if .Wants}}<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
|
||||
{{else}}<p>no difference: every want is what the controller sends.</p>{{end}}</body></html>`))
|
||||
|
||||
// Web is the read-only web view of the same rows, refreshed by the browser.
|
||||
@@ -387,7 +417,7 @@ func Web(m *Model) http.Handler {
|
||||
}
|
||||
if r.URL.Path == "/wants" {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
wantsPage.Execute(w, map[string]any{"Head": differenceHead, "Differences": differenceText(m.Snapshot())})
|
||||
wantsPage.Execute(w, map[string]any{"Head": wantHead, "Wants": wantText(m.Snapshot())})
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/" {
|
||||
@@ -410,7 +440,7 @@ func Web(m *Model) http.Handler {
|
||||
for _, u := range snap.Unreadable {
|
||||
bad = append(bad, []string{u.Bucket, u.Key, u.Error})
|
||||
}
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Differences": differenceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "DifferenceHead": differenceHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Wants": wantText(snap), "Source": sourceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "WantHead": wantHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ type Entry struct {
|
||||
Value []byte
|
||||
At time.Time
|
||||
Deleted bool
|
||||
// Revision is the bucket revision of the change, when the watch gives one (the source revision is the gate's source key's).
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// Row is one module on one node.
|
||||
@@ -66,27 +68,59 @@ type Snapshot struct {
|
||||
Fingerprints []FingerprintRow `json:"fingerprints"`
|
||||
// Asks is every ask the gate holds that has no answer yet: what waits for the operator's Approve.
|
||||
Asks []AskRow `json:"asks"`
|
||||
// Differences is every difference between what the assigner wants a node to run and what the controller sends, with the rule that
|
||||
// explains it or unexplained (novox/hq ADR 0528 clause 4, issue 1297 B8).
|
||||
Differences []DifferenceRow `json:"differences"`
|
||||
// Wants is every want of the assigner beside the have of its node and where it stands (novox/hq ADR 0539 §8, issue 1906 D7).
|
||||
Wants []WantRow `json:"wants"`
|
||||
// Source is the state of the mesh's source revision, derived from the wants and the haves (ADR 0429 §1): recorded by no part.
|
||||
// Nil when the gate holds no source yet.
|
||||
Source *SourceRow `json:"source,omitempty"`
|
||||
// Backlog is, for each language and kind of build ask with one waiting or held, the asks waiting for a node, the asks a node
|
||||
// holds and the oldest waiting ask's age (issue 1887 A8a).
|
||||
Backlog []BacklogRow `json:"backlog"`
|
||||
}
|
||||
|
||||
// DifferenceRow is a module on a node whose want and send differ: the fingerprint wanted (empty when the node has no want,
|
||||
// WantRemoved when the want is removed), the fingerprint of the build the controller's assignment names (empty when it sends
|
||||
// none; SentBuild names that build), and the rule that explains it (Rule 1 to 6) or Unexplained.
|
||||
type DifferenceRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Want string `json:"want,omitempty"`
|
||||
WantRemoved bool `json:"want-removed,omitempty"`
|
||||
Sent string `json:"sent,omitempty"`
|
||||
SentBuild string `json:"sent-build,omitempty"`
|
||||
Rule int `json:"rule,omitempty"`
|
||||
Unexplained bool `json:"unexplained,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
// The states of a want against the have of its node.
|
||||
const (
|
||||
WantApplied = "applied" // the node runs the want's fingerprint, running and healthy
|
||||
WantPending = "pending" // the node has not yet run the want's fingerprint, or not healthy yet
|
||||
WantFailed = "failed" // the node runs the want's fingerprint and its health check failed
|
||||
WantHeldBack = "held back" // the assigner holds the node back from the newest fingerprint (Rule), or set it back
|
||||
WantRemoved = "removed" // the node's want is removed: the module is no longer assigned to it
|
||||
)
|
||||
|
||||
// WantRow is a want of the assigner beside the have of its node: the fingerprint wanted, the one the node runs with its phase and
|
||||
// health, the state of the two, and for a want held back the newest fingerprint, the rule and the assigner's words.
|
||||
type WantRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Want string `json:"want,omitempty"`
|
||||
Have string `json:"have,omitempty"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
Health string `json:"health,omitempty"`
|
||||
State string `json:"state"`
|
||||
Newest string `json:"newest,omitempty"`
|
||||
Rule int `json:"rule,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
// Judgement is the controller's judgement of the move to the composition the want names (recreates, moves-data, neither or
|
||||
// cannot-judge), and Judged its words; empty when the controller wrote none (ADR 0539 §14).
|
||||
Judgement string `json:"judgement,omitempty"`
|
||||
Judged string `json:"judged,omitempty"`
|
||||
}
|
||||
|
||||
// The states of a source revision (ADR 0429 §1).
|
||||
const (
|
||||
SourceDelivered = "delivered"
|
||||
SourceNotDelivered = "not delivered"
|
||||
)
|
||||
|
||||
// SourceRow is the mesh's source revision (the revision of the gate's source key) and whether it is delivered: every want equals its
|
||||
// module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is assigned to
|
||||
// has a want. Waiting names each module on a node that is not there yet, and why.
|
||||
type SourceRow struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
State string `json:"state"`
|
||||
Waiting []string `json:"waiting,omitempty"`
|
||||
// Wants is how many wants of modules at the indexer were looked at; a revision with none is delivered of nothing.
|
||||
Wants int `json:"wants"`
|
||||
}
|
||||
|
||||
// AskRow is an open ask (novox/hq ADR 0437 §10): the gate's ask with no answer under its key in the teller's answers.
|
||||
@@ -178,22 +212,31 @@ type UnreadableRow struct {
|
||||
|
||||
// Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values.
|
||||
type Model struct {
|
||||
mu sync.Mutex
|
||||
assigns map[string]assignment
|
||||
builds map[string]buildEntry // by <module>.build.<id>
|
||||
groups map[string]groupEntry // the indexer's candidates, by group
|
||||
modules map[string]moduleEntry // the indexer's modules, by module
|
||||
keys map[string]keyEntry // the builder's build keys, by <module>.<fingerprint>
|
||||
asks map[string]askRecord // the gate's asks, by key
|
||||
diffs map[string]diffEntry // the assigner's differences, by <node>.<module>
|
||||
aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key
|
||||
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
||||
writer Writer
|
||||
backlog *Backlog
|
||||
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
||||
newest map[string]string // module -> id
|
||||
haves map[string]haveEntry
|
||||
nodes map[string]nodeEntry
|
||||
mu sync.Mutex
|
||||
assigns map[string]assignment
|
||||
builds map[string]buildEntry // by <module>.build.<id>
|
||||
groups map[string]groupEntry // the indexer's candidates, by group
|
||||
modules map[string]moduleEntry // the indexer's modules, by module
|
||||
keys map[string]keyEntry // the builder's build keys, by <module>.<fingerprint>
|
||||
asks map[string]askRecord // the gate's asks, by key
|
||||
wants map[string]wantEntry // the assigner's wants, by <node>.<module>
|
||||
source *sourceEntry // the gate's source key
|
||||
deliveredEvent func(DeliveredEvent)
|
||||
motionEvent func(MotionEvent)
|
||||
told map[string]bool // the motion events said, by their ids
|
||||
initial map[string]bool // the buckets whose watch handed over what stood at its start
|
||||
said uint64 // the source revision whose delivery was said
|
||||
composed map[string]composedEntry // the controller's digests and judgements, by <node>.<module>.<hex>
|
||||
aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key
|
||||
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
||||
writer Writer
|
||||
backlog *Backlog
|
||||
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
||||
answerOf map[string]delivery.Answer // by key: the teller's answer whole, when readable (holds.go)
|
||||
holds map[string]heldEntry // the assigner's holds, by held key (holds.go)
|
||||
newest map[string]string // module -> id
|
||||
haves map[string]haveEntry
|
||||
nodes map[string]nodeEntry
|
||||
// the watchman's buckets, by key
|
||||
presence map[string]presenceEntry
|
||||
stuck map[string]stuckEntry
|
||||
@@ -256,9 +299,22 @@ type keyEntry struct {
|
||||
err string
|
||||
}
|
||||
|
||||
type diffEntry struct {
|
||||
value delivery.Difference
|
||||
type wantEntry struct {
|
||||
value delivery.Want
|
||||
err string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
type composedEntry struct {
|
||||
value delivery.ComposedDigest
|
||||
err string
|
||||
}
|
||||
|
||||
type sourceEntry struct {
|
||||
revision uint64
|
||||
group string
|
||||
heads map[string]string
|
||||
err string
|
||||
}
|
||||
|
||||
type assignerAskRecord struct {
|
||||
@@ -290,18 +346,64 @@ type nodeEntry struct {
|
||||
|
||||
// New is an empty model.
|
||||
func New() *Model {
|
||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, diffs: map[string]diffEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{},
|
||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{},
|
||||
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
||||
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
|
||||
}
|
||||
|
||||
// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed.
|
||||
// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed. When the change makes
|
||||
// the source revision delivered, the event of it is said once (SetDeliveredEvent), after the model's lock is released.
|
||||
func (m *Model) Apply(e Entry) {
|
||||
m.mu.Lock()
|
||||
m.apply(e)
|
||||
event := m.deliveredNow()
|
||||
say := m.deliveredEvent
|
||||
motion, sayMotion := m.motionNow(e), m.motionEvent
|
||||
m.mu.Unlock()
|
||||
if event != nil && say != nil {
|
||||
say(*event)
|
||||
}
|
||||
if sayMotion != nil {
|
||||
for _, ev := range motion {
|
||||
sayMotion(ev)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeliveredEvent is what the view says when the source revision is delivered (event `delivered` of the view, issue 1906 D9, ADR
|
||||
// 0539 §17): the revision of the gate's source key, the group that moved it and every repository's merged head at it. A
|
||||
// consumer takes every earlier revision as covered by it, and takes the same revision twice as once (after a restart of the view).
|
||||
type DeliveredEvent struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
Group string `json:"group"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
}
|
||||
|
||||
// SetDeliveredEvent sets where the event of a delivered source revision goes; nothing is said until it is set.
|
||||
func (m *Model) SetDeliveredEvent(say func(DeliveredEvent)) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.deliveredEvent = say
|
||||
}
|
||||
|
||||
// deliveredNow is the event of the source revision when it is delivered now and has not been said, or nil. A revision that went
|
||||
// bad after it was delivered and recovered is not said again.
|
||||
func (m *Model) deliveredNow() *DeliveredEvent {
|
||||
if m.deliveredEvent == nil || m.source == nil || m.source.err != "" || m.said == m.source.revision {
|
||||
return nil
|
||||
}
|
||||
row := m.sourceRow()
|
||||
if row == nil || row.State != SourceDelivered || row.Wants == 0 {
|
||||
return nil
|
||||
}
|
||||
m.said = m.source.revision
|
||||
return &DeliveredEvent{Revision: m.source.revision, Group: m.source.group, Heads: m.source.heads}
|
||||
}
|
||||
|
||||
func (m *Model) apply(e Entry) {
|
||||
switch e.Bucket {
|
||||
case delivery.HaveBucket:
|
||||
node, module, isNode, err := delivery.ParseHaveKey(e.Key)
|
||||
node, _, isNode, err := delivery.ParseHaveKey(e.Key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -314,7 +416,7 @@ func (m *Model) Apply(e Entry) {
|
||||
m.nodes[node] = nodeEntry{node: n, err: errText(err), at: e.At}
|
||||
return
|
||||
}
|
||||
key := node + "." + module
|
||||
key := e.Key // the bucket's own key: a module name with a dot is escaped in it (issue 1150)
|
||||
if e.Deleted {
|
||||
delete(m.haves, key)
|
||||
return
|
||||
@@ -385,16 +487,45 @@ func (m *Model) Apply(e Entry) {
|
||||
if a, err := delivery.DecodeViewAnswer(e.Value); err == nil {
|
||||
m.chosen[askKey] = a
|
||||
}
|
||||
case delivery.AssignerDifferencesBucket:
|
||||
if e.Deleted {
|
||||
delete(m.diffs, e.Key)
|
||||
case delivery.AssignerWantsBucket:
|
||||
if m.applyHeld(e) {
|
||||
return
|
||||
}
|
||||
d, err := delivery.DecodeDifference(e.Value)
|
||||
m.diffs[e.Key] = diffEntry{value: d, err: errText(err)}
|
||||
node, module, err := delivery.ParseWantKey(e.Key)
|
||||
if err != nil {
|
||||
return // the node's own key is not a module's want
|
||||
}
|
||||
key, kerr := delivery.HaveKey(node, module)
|
||||
if kerr != nil {
|
||||
return
|
||||
}
|
||||
if e.Deleted {
|
||||
delete(m.wants, key)
|
||||
return
|
||||
}
|
||||
w, err := delivery.DecodeWant(e.Value)
|
||||
m.wants[key] = wantEntry{value: w, err: errText(err), at: e.At}
|
||||
case delivery.ControllerComposedBucket:
|
||||
if e.Deleted {
|
||||
delete(m.composed, e.Key)
|
||||
return
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(e.Value)
|
||||
m.composed[e.Key] = composedEntry{value: d, err: errText(err)}
|
||||
case delivery.SourceBucket:
|
||||
if e.Key != delivery.SourceMainKey {
|
||||
return
|
||||
}
|
||||
if e.Deleted {
|
||||
m.source = nil
|
||||
return
|
||||
}
|
||||
src, err := delivery.DecodeSource(e.Value)
|
||||
m.source = &sourceEntry{revision: e.Revision, group: src.Group, heads: src.Heads, err: errText(err)}
|
||||
case delivery.AnswersBucket:
|
||||
if e.Deleted {
|
||||
delete(m.answers, e.Key)
|
||||
delete(m.answerOf, e.Key)
|
||||
return
|
||||
}
|
||||
// The answer's warrant is `<router id>@<time>`: it ends that asking, and only that one (a later asking of the key is
|
||||
@@ -402,6 +533,9 @@ func (m *Model) Apply(e Entry) {
|
||||
id := ""
|
||||
if a, err := delivery.DecodeAnswer(e.Value); err == nil {
|
||||
id, _, _ = strings.Cut(a.Warrant, "@")
|
||||
m.answerOf[e.Key] = a
|
||||
} else {
|
||||
delete(m.answerOf, e.Key)
|
||||
}
|
||||
m.answers[e.Key] = id
|
||||
case delivery.BuilderBuildsBucket:
|
||||
@@ -487,7 +621,17 @@ func (m *Model) snapshot() Snapshot {
|
||||
r.RequiredBy = a.RequiredBy
|
||||
}
|
||||
var waits []string
|
||||
h, has := m.haves[key]
|
||||
hk, herr := delivery.HaveKey(a.Node, a.Module)
|
||||
if herr != nil {
|
||||
hk = key
|
||||
}
|
||||
h, has := m.haves[hk]
|
||||
// The controller no longer writes the build of an assignment (R6): the row is the want and the have of the node.
|
||||
w, wanted := m.wants[hk]
|
||||
wanted = wanted && w.err == "" && !w.value.Removed && w.value.FP != ""
|
||||
if wanted {
|
||||
r.Assigned = w.value.FP
|
||||
}
|
||||
switch {
|
||||
case !has:
|
||||
waits = append(waits, "no have yet")
|
||||
@@ -495,8 +639,15 @@ func (m *Model) snapshot() Snapshot {
|
||||
r.Since, r.Error = h.at, h.err
|
||||
default:
|
||||
r.Have, r.Phase, r.Health = h.have.Build, string(h.have.Phase), string(h.have.Health)
|
||||
if r.Have == "" {
|
||||
r.Have = h.have.FP
|
||||
}
|
||||
r.Since, r.Detail = h.at, h.have.Detail
|
||||
if a.Build != "" && h.have.Build != a.Build {
|
||||
if wanted {
|
||||
if h.have.FP != w.value.FP {
|
||||
waits = append(waits, "the node has not applied "+w.value.FP)
|
||||
}
|
||||
} else if a.Build != "" && h.have.Build != a.Build {
|
||||
waits = append(waits, "the node has not applied "+a.Build)
|
||||
}
|
||||
}
|
||||
@@ -509,10 +660,13 @@ func (m *Model) snapshot() Snapshot {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
for key, h := range m.haves {
|
||||
if a, ok := m.assigns[key]; ok && a.open() {
|
||||
node, module, _, perr := delivery.ParseHaveKey(key)
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := m.assigns[node+"."+module]; ok && a.open() {
|
||||
continue
|
||||
}
|
||||
node, module, _, _ := delivery.ParseHaveKey(key)
|
||||
r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err}
|
||||
if h.err == "" && h.have.Phase == delivery.PhaseRemoved {
|
||||
// The node-engine says it took the module away: plainly removed, not left behind.
|
||||
@@ -520,6 +674,9 @@ func (m *Model) snapshot() Snapshot {
|
||||
}
|
||||
if h.err == "" {
|
||||
r.Have, r.Phase, r.Health, r.Detail = h.have.Build, string(h.have.Phase), string(h.have.Health), h.have.Detail
|
||||
if r.Have == "" {
|
||||
r.Have = h.have.FP
|
||||
}
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
@@ -585,9 +742,9 @@ func (m *Model) snapshot() Snapshot {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerAsksBucket, Key: key, Error: a.err})
|
||||
}
|
||||
}
|
||||
for key, d := range m.diffs {
|
||||
if d.err != "" {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerDifferencesBucket, Key: key, Error: d.err})
|
||||
for key, w := range m.wants {
|
||||
if w.err != "" {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerWantsBucket, Key: key, Error: w.err})
|
||||
}
|
||||
}
|
||||
sort.Slice(offline, func(i, j int) bool {
|
||||
@@ -601,7 +758,7 @@ func (m *Model) snapshot() Snapshot {
|
||||
return unreadable[i].Bucket+"."+unreadable[i].Key < unreadable[j].Bucket+"."+unreadable[j].Key
|
||||
})
|
||||
return Snapshot{Rows: rows, Nodes: nodes, Offline: offline, Stuck: stuck, Unreadable: unreadable,
|
||||
Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Differences: m.differenceRows()}
|
||||
Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Wants: m.wantRows(), Source: m.sourceRow()}
|
||||
}
|
||||
|
||||
// groupRows are the readable candidate groups, ordered by group, their pull requests by repository and number.
|
||||
@@ -840,20 +997,38 @@ func (m *Model) Answer(askKey, option, words string) error {
|
||||
return w.Put(key, body)
|
||||
}
|
||||
|
||||
// differenceRows are the readable differences, ordered by node and module.
|
||||
func (m *Model) differenceRows() []DifferenceRow {
|
||||
out := []DifferenceRow{}
|
||||
for key, d := range m.diffs {
|
||||
if d.err != "" {
|
||||
// wantRows are the readable wants beside the haves of their nodes, ordered by node and module.
|
||||
func (m *Model) wantRows() []WantRow {
|
||||
out := []WantRow{}
|
||||
for key, w := range m.wants {
|
||||
if w.err != "" {
|
||||
continue
|
||||
}
|
||||
node, module, err := delivery.ParseDifferenceKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
node, module := splitKey(key)
|
||||
v := w.value
|
||||
row := WantRow{Node: node, Module: module, Want: v.FP, Newest: v.Newest, Rule: v.Rule, Words: v.Words}
|
||||
h, has := m.haves[key]
|
||||
if has && h.err == "" {
|
||||
row.Have, row.Phase, row.Health = h.have.FP, string(h.have.Phase), string(h.have.Health)
|
||||
}
|
||||
v := d.value
|
||||
out = append(out, DifferenceRow{Node: node, Module: module, Want: v.Want, WantRemoved: v.WantRemoved, Sent: v.Sent, SentBuild: v.SentBuild,
|
||||
Rule: v.Rule, Unexplained: v.Unexplained, Words: v.Words})
|
||||
if ckey, err := delivery.ComposedKey(node, module, v.FP); err == nil && !v.Removed {
|
||||
if c, ok := m.composed[ckey]; ok && c.err == "" {
|
||||
row.Judgement, row.Judged = string(c.value.Judgement), c.value.Words
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case v.Removed:
|
||||
row.State = WantRemoved
|
||||
case v.Rule != 0 || v.Newest != "":
|
||||
row.State = WantHeldBack
|
||||
case row.Have == v.FP && h.have.Health == delivery.HealthFailed:
|
||||
row.State = WantFailed
|
||||
case row.Have == v.FP && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK:
|
||||
row.State = WantApplied
|
||||
default:
|
||||
row.State = WantPending
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Node != out[j].Node {
|
||||
@@ -863,3 +1038,257 @@ func (m *Model) differenceRows() []DifferenceRow {
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// sourceRow derives the state of the source revision the gate holds (ADR 0429 §1; recorded by no part): delivered when every want
|
||||
// equals its module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is
|
||||
// assigned to has a want. It is nil when the gate holds no readable source.
|
||||
func (m *Model) sourceRow() *SourceRow {
|
||||
if m.source == nil || m.source.err != "" {
|
||||
return nil
|
||||
}
|
||||
waiting := []string{}
|
||||
short := func(fp string) string {
|
||||
fp = strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
if len(fp) > 12 {
|
||||
fp = fp[:12]
|
||||
}
|
||||
return fp
|
||||
}
|
||||
looked := 0
|
||||
for key, w := range m.wants {
|
||||
node, module := splitKey(key)
|
||||
mod, ok := m.modules[module]
|
||||
if w.err != "" || w.value.Removed || !ok || mod.err != "" || mod.value.FP2 == "" {
|
||||
continue
|
||||
}
|
||||
looked++
|
||||
h := m.haves[key]
|
||||
switch {
|
||||
case w.value.FP != mod.value.FP2:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s wants %s, not the module's %s", module, node, short(w.value.FP), short(mod.value.FP2)))
|
||||
case h.err != "" || h.have.FP != w.value.FP:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s does not yet run %s", module, node, short(w.value.FP)))
|
||||
case h.have.Health == delivery.HealthFailed:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s runs %s and its health check failed", module, node, short(w.value.FP)))
|
||||
case h.have.Phase != delivery.PhaseRunning || h.have.Health != delivery.HealthOK:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet running and healthy", module, node, short(w.value.FP)))
|
||||
}
|
||||
}
|
||||
for key, a := range m.assigns {
|
||||
if !a.open() {
|
||||
continue
|
||||
}
|
||||
wk, werr := delivery.HaveKey(a.Node, a.Module)
|
||||
if werr != nil {
|
||||
wk = key
|
||||
}
|
||||
if _, ok := m.wants[wk]; ok {
|
||||
continue
|
||||
}
|
||||
if mod, ok := m.modules[a.Module]; ok && mod.err == "" && mod.value.FP2 != "" {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s has no want yet", a.Module, a.Node))
|
||||
}
|
||||
}
|
||||
sort.Strings(waiting)
|
||||
row := &SourceRow{Revision: m.source.revision, State: SourceDelivered, Wants: looked}
|
||||
if len(waiting) > 0 {
|
||||
row.State, row.Waiting = SourceNotDelivered, waiting
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
// splitKey is the node and the module (unescaped, issue 1150) of a model key, which is the have bucket's key form.
|
||||
func splitKey(key string) (node, module string) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
node, module, _ = strings.Cut(key, ".")
|
||||
}
|
||||
return node, module
|
||||
}
|
||||
|
||||
// The events of a module in motion (issue 1906 D10, for the messenger, ADR 0308): with no plan to say it, the view says that a
|
||||
// module's want on a node differs from its have (moving), that the have reached the want running and healthy (settled), or that it
|
||||
// is held or failed or stopped moving past the watchman's bound (stuck). Each is said once for a node, a module and a composition.
|
||||
const (
|
||||
MotionMoving = "moving"
|
||||
MotionSettled = "settled"
|
||||
MotionStuck = "stuck"
|
||||
// MotionBoundSeconds is how long a node may take to apply a want before the watchman says it stuck (watch.StepBound).
|
||||
MotionBoundSeconds = 1800
|
||||
)
|
||||
|
||||
// MotionBody is the body of the three events. Outcome and Words are those of settled and stuck: settled is "running"; stuck is
|
||||
// "failed" (a health check failed) or "stopped" (the module is held, or the watchman says the want stopped moving past its bound),
|
||||
// with the have's own words or the watchman's in Words.
|
||||
type MotionBody struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
FromFP string `json:"from_fp"`
|
||||
ToFP string `json:"to_fp"`
|
||||
Composed string `json:"composed"`
|
||||
Since time.Time `json:"since"`
|
||||
BoundSeconds int `json:"bound_seconds"`
|
||||
// At is when the have or the want last changed, the time the event is about.
|
||||
At time.Time `json:"at"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
// MotionEvent is an event to say: its type (moving, settled or stuck), its id (the bus's dedup id and the event id header, one for
|
||||
// a type, a node, a module and a composition) and its body.
|
||||
type MotionEvent struct {
|
||||
Type string
|
||||
ID string
|
||||
Body MotionBody
|
||||
}
|
||||
|
||||
// SetMotionEvent sets where the events of modules in motion go; nothing is said until it is set.
|
||||
func (m *Model) SetMotionEvent(say func(MotionEvent)) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.motionEvent = say
|
||||
}
|
||||
|
||||
// motionNow is the events to say now about the node and module the change touched: a want, a have, or the watchman's stuck key of a
|
||||
// want.
|
||||
func (m *Model) motionNow(e Entry) []MotionEvent {
|
||||
if m.motionEvent == nil || !m.motionReady() {
|
||||
return nil
|
||||
}
|
||||
var hk string
|
||||
switch e.Bucket {
|
||||
case delivery.AssignerWantsBucket:
|
||||
node, module, err := delivery.ParseWantKey(e.Key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
hk, _ = delivery.HaveKey(node, module)
|
||||
case delivery.HaveBucket:
|
||||
if _, _, isNode, err := delivery.ParseHaveKey(e.Key); err != nil || isNode {
|
||||
return nil
|
||||
}
|
||||
hk = e.Key
|
||||
case delivery.StuckBucket:
|
||||
st, ok := m.stuck[e.Key]
|
||||
if !ok || st.err != "" || st.value.Bucket != delivery.AssignerWantsBucket {
|
||||
return nil
|
||||
}
|
||||
node, module, err := delivery.ParseWantKey(st.value.Key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
hk, _ = delivery.HaveKey(node, module)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return m.motionOf(hk)
|
||||
}
|
||||
|
||||
// motionReady says whether the view has read what stands in the buckets motion is derived from (the wants, the haves and the
|
||||
// watchman's stuck keys): before, a want read ahead of its have would be said moving though the node runs it.
|
||||
func (m *Model) motionReady() bool {
|
||||
for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
|
||||
if !m.initial[b] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Initial tells the model that the watch of a bucket has handed over every key that stood at its start. When the buckets motion is
|
||||
// derived from have all done so, the events of the modules in motion at that moment are said (the ids deduplicate what a restart
|
||||
// of the view says again).
|
||||
func (m *Model) Initial(bucket string) {
|
||||
m.mu.Lock()
|
||||
was := m.motionReady()
|
||||
m.initial[bucket] = true
|
||||
var out []MotionEvent
|
||||
if !was && m.motionReady() && m.motionEvent != nil {
|
||||
keys := make([]string, 0, len(m.wants))
|
||||
for k := range m.wants {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
out = append(out, m.motionOf(k)...)
|
||||
}
|
||||
}
|
||||
say := m.motionEvent
|
||||
m.mu.Unlock()
|
||||
for _, ev := range out {
|
||||
say(ev)
|
||||
}
|
||||
}
|
||||
|
||||
// motionOf is the events to say now about the node and module of a model key, from their want, have and stuck key.
|
||||
func (m *Model) motionOf(hk string) []MotionEvent {
|
||||
w, ok := m.wants[hk]
|
||||
if !ok || w.err != "" || w.value.Removed || w.value.FP == "" {
|
||||
return nil
|
||||
}
|
||||
node, module := splitKey(hk)
|
||||
h, has := m.haves[hk]
|
||||
has = has && h.err == ""
|
||||
body := MotionBody{Node: node, Module: module, ToFP: w.value.FP, Composed: w.value.Composed, Since: w.at, BoundSeconds: MotionBoundSeconds}
|
||||
body.At = w.at
|
||||
if has {
|
||||
body.FromFP = h.have.FP
|
||||
if h.at.After(body.At) {
|
||||
body.At = h.at
|
||||
}
|
||||
}
|
||||
id := func(kind string) string {
|
||||
return kind + "." + node + "." + module + "." + strings.TrimPrefix(w.value.Composed, "sha256:")
|
||||
}
|
||||
applied := has && h.have.FP == w.value.FP && (h.have.Composed == "" || h.have.Composed == w.value.Composed)
|
||||
var out []MotionEvent
|
||||
say := func(kind string, b MotionBody) {
|
||||
if m.told[id(kind)] {
|
||||
return
|
||||
}
|
||||
m.told[id(kind)] = true
|
||||
out = append(out, MotionEvent{Type: kind, ID: id(kind), Body: b})
|
||||
}
|
||||
switch {
|
||||
case applied && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK:
|
||||
if m.told[id(MotionMoving)] {
|
||||
b := body
|
||||
b.Outcome = "running"
|
||||
say(MotionSettled, b)
|
||||
}
|
||||
return out
|
||||
case !m.told[id(MotionMoving)]:
|
||||
say(MotionMoving, body)
|
||||
}
|
||||
stop := body
|
||||
switch {
|
||||
case has && h.have.Phase == delivery.PhaseHeld:
|
||||
stop.Outcome, stop.Words = "stopped", h.have.Detail
|
||||
case has && (h.have.Health == delivery.HealthFailed || h.have.Phase == delivery.PhaseFailed) && h.have.FP == w.value.FP:
|
||||
stop.Outcome, stop.Words = "failed", h.have.Detail
|
||||
default:
|
||||
if s, ok := m.stuckWant(w.value, node, module); ok {
|
||||
stop.Outcome, stop.Words = "stopped", s
|
||||
} else {
|
||||
return out
|
||||
}
|
||||
}
|
||||
say(MotionStuck, stop)
|
||||
return out
|
||||
}
|
||||
|
||||
// stuckWant is the watchman's words for the want of module on node, when it says the want stopped moving and not yet moved.
|
||||
func (m *Model) stuckWant(_ delivery.Want, node, module string) (string, bool) {
|
||||
wk, err := delivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
sk, err := delivery.StuckKey(delivery.AssignerWantsBucket, wk)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if st, ok := m.stuck[sk]; ok && st.err == "" && !st.value.Moved {
|
||||
return st.value.Waited, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
func wantE(t *testing.T, node, module string, w delivery.Want) Entry {
|
||||
t.Helper()
|
||||
key, err := delivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !w.Removed {
|
||||
w.Composed = "sha256:" + strings.Repeat("c", 64)
|
||||
}
|
||||
v, err := delivery.EncodeWant(w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AssignerWantsBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func haveFP(t *testing.T, node, module, fp string, health delivery.Health) Entry {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: health})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.HaveBucket, Key: key, Value: b, At: t0}
|
||||
}
|
||||
|
||||
func moduleE(t *testing.T, module, fp string) Entry {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/" + module, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.ModulesBucket, Key: module, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func sourceE(t *testing.T, revision uint64) Entry {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.SourceBucket, Key: delivery.SourceMainKey, Value: v, At: t0, Revision: revision}
|
||||
}
|
||||
|
||||
// 1906/D7 AC7: every want is a row beside the have of its node, with where it stands.
|
||||
func TestEveryWantIsARowAgainstItsHave(t *testing.T) {
|
||||
m := New()
|
||||
fill(m,
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK),
|
||||
wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed),
|
||||
wantE(t, "g14", "mail", delivery.Want{FP: fpOld, Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"}),
|
||||
wantE(t, "g14", "old", delivery.Want{Removed: true}),
|
||||
wantE(t, "novox", "web", delivery.Want{FP: fpNew}),
|
||||
)
|
||||
got := map[string]WantRow{}
|
||||
for _, r := range m.Snapshot().Wants {
|
||||
got[r.Node+"."+r.Module] = r
|
||||
}
|
||||
for key, state := range map[string]string{
|
||||
"ace.web": WantApplied, "ace.mail": WantPending, "g14.web": WantFailed, "g14.mail": WantHeldBack, "g14.old": WantRemoved, "novox.web": WantPending,
|
||||
} {
|
||||
if got[key].State != state {
|
||||
t.Errorf("%s = %+v, want state %s", key, got[key], state)
|
||||
}
|
||||
}
|
||||
if r := got["g14.mail"]; r.Rule != delivery.RuleCanary || r.Newest != fpNew || r.Words == "" {
|
||||
t.Errorf("the held-back row keeps its rule and words: %+v", r)
|
||||
}
|
||||
if r := got["ace.mail"]; r.Want != fpNew || r.Have != fpOld {
|
||||
t.Errorf("a pending row shows both: %+v", r)
|
||||
}
|
||||
// a want taken away leaves the list
|
||||
fill(m, Entry{Bucket: delivery.AssignerWantsBucket, Key: "want.novox.web", Deleted: true})
|
||||
if n := len(m.Snapshot().Wants); n != 5 {
|
||||
t.Errorf("wants = %d after a delete, want 5", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D7 AC7: a source revision is delivered when every want equals its module's fingerprint and every have equals its
|
||||
// want, healthy; one failed have shows it not delivered and names the module.
|
||||
func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew),
|
||||
assign("ace", "web", "b-1", ""), assign("g14", "web", "b-1", ""),
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpNew, delivery.HealthOK))
|
||||
s := m.Snapshot().Source
|
||||
if s == nil || s.Revision != 14355 || s.State != SourceDelivered || len(s.Waiting) != 0 {
|
||||
t.Fatalf("source = %+v, want revision 14355 delivered", s)
|
||||
}
|
||||
fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthFailed))
|
||||
s = m.Snapshot().Source
|
||||
if s.State != SourceNotDelivered || len(s.Waiting) != 1 || !strings.Contains(s.Waiting[0], "web") || !strings.Contains(s.Waiting[0], "g14") {
|
||||
t.Errorf("a failed have: %+v", s)
|
||||
}
|
||||
fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "mail", delivery.Want{FP: fpOld}))
|
||||
s = m.Snapshot().Source
|
||||
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") {
|
||||
t.Errorf("a want that is not the module's fingerprint: %+v", s)
|
||||
}
|
||||
fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), assign("novox", "web", "b-1", ""))
|
||||
s = m.Snapshot().Source
|
||||
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") {
|
||||
t.Errorf("a node assigned the module with no want: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoSourceRevisionHeldMeansNoStateIsDerived(t *testing.T) {
|
||||
if s := New().Snapshot().Source; s != nil {
|
||||
t.Errorf("source = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWantsPageSaysWantsAgainstHaves(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
text := wantText(m.Snapshot())
|
||||
if len(text) != 1 || text[0][0] != "ace" || text[0][1] != "web" || text[0][2] != fpNew || text[0][3] != fpOld || text[0][4] != WantPending || text[0][5] != "none" {
|
||||
t.Errorf("text = %v", text)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D7 (ADR 0539 §14): the controller's judgement of the move a want names is shown beside it, with its words.
|
||||
func TestTheJudgementOfTheMoveIsShownBesideTheWant(t *testing.T) {
|
||||
m := New()
|
||||
key, err := delivery.ComposedKey("ace", "web", fpNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.ControllerComposedBucket, Key: key, Value: v, At: t0})
|
||||
rows := m.Snapshot().Wants
|
||||
if len(rows) != 1 || rows[0].Judgement != string(delivery.JudgedCannot) || !strings.Contains(rows[0].Judged, "gone") {
|
||||
t.Fatalf("rows = %+v", rows)
|
||||
}
|
||||
text := wantText(m.Snapshot())
|
||||
if len(text[0]) != 7 || text[0][5] != "cannot-judge" {
|
||||
t.Errorf("text = %v", text)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D9 (asked by the lead for issue 1946): the view says, once for each source revision, that it is delivered: the event the
|
||||
// issue tracker derives "fix delivered" from, in place of the gate's states that go in release 6 (ADR 0539 §17).
|
||||
func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) {
|
||||
m := New()
|
||||
var said []DeliveredEvent
|
||||
m.SetDeliveredEvent(func(e DeliveredEvent) { said = append(said, e) })
|
||||
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), assign("ace", "web", "b-1", ""),
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said %v before any have", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed))
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said %v for a failed have", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if len(said) != 1 || said[0].Revision != 14355 || said[0].Group != "g" || said[0].Heads["novox/web"] == "" {
|
||||
t.Fatalf("said = %+v, want revision 14355 once with its group and heads", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if len(said) != 1 {
|
||||
t.Errorf("said again for the same revision: %v", said)
|
||||
}
|
||||
// a newer revision is delivered when its wants and haves are; a revision that goes bad and good again is not said twice
|
||||
fill(m, sourceE(t, 14356))
|
||||
if len(said) != 2 || said[1].Revision != 14356 {
|
||||
t.Errorf("the next revision: %v", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed), haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if len(said) != 2 {
|
||||
t.Errorf("said again after a have failed and recovered: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (view notes): the controller no longer writes the build of an assignment (C6), so a row reads the want and the
|
||||
// have: the row shows the fingerprint wanted and waits while the node runs another.
|
||||
func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, assign("ace", "web", "", ""), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
r := find(t, m.Snapshot(), "ace", "web")
|
||||
if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") {
|
||||
t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if r := find(t, m.Snapshot(), "ace", "web"); r.Waits != "" || r.Have != fpNew {
|
||||
t.Errorf("row = %+v, want no wait once the have names the want's fingerprint", r)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (view notes, issue 1150): a module whose name holds a dot has its have under the escaped key; the row finds it,
|
||||
// and a have nothing is assigned to keeps the module's name.
|
||||
func TestADottedModuleNameIsFoundAndKept(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, assign("ace", "novox.be", "", ""), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK),
|
||||
haveFP(t, "ace", "old.name", fpNew, delivery.HealthOK))
|
||||
s := m.Snapshot()
|
||||
if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Waits != "" {
|
||||
t.Errorf("the dotted module's row = %+v", r)
|
||||
}
|
||||
if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind {
|
||||
t.Errorf("the left-behind dotted module = %+v", r)
|
||||
}
|
||||
if rows := s.Wants; len(rows) != 1 || rows[0].Module != "novox.be" || rows[0].State != WantApplied {
|
||||
t.Errorf("wants = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D10 (asked by the lead for the messenger, ADR 0308): with no plans to say that a module is in motion, the view says it: `moving`
|
||||
// when a module's want on a node differs from its have, `settled` when the have reaches the want running and healthy, `stuck` when it
|
||||
// is held or failed or the watchman says it stopped moving; each once for a node, a module and a composition.
|
||||
func motionLog(m *Model) *[]MotionEvent {
|
||||
var said []MotionEvent
|
||||
m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) })
|
||||
for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
|
||||
m.Initial(b)
|
||||
}
|
||||
return &said
|
||||
}
|
||||
|
||||
func types(said []MotionEvent) string {
|
||||
var out []string
|
||||
for _, e := range said {
|
||||
out = append(out, e.Type)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func TestAModuleInMotionIsSaidMovingThenSettledOnceEach(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("said %v for a have with no want", types(*said))
|
||||
}
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if types(*said) != "moving" {
|
||||
t.Fatalf("said %q, want moving", types(*said))
|
||||
}
|
||||
mv := (*said)[0]
|
||||
if mv.Body.Node != "ace" || mv.Body.Module != "web" || mv.Body.FromFP != fpOld || mv.Body.ToFP != fpNew || mv.Body.Composed != "sha256:"+strings.Repeat("c", 64) ||
|
||||
mv.Body.Since.IsZero() || mv.Body.At.IsZero() || mv.Body.BoundSeconds != 1800 || mv.ID != "moving.ace.web."+strings.Repeat("c", 64) {
|
||||
t.Errorf("moving = %+v", mv)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if types(*said) != "moving" {
|
||||
t.Errorf("said %q after the same state again", types(*said))
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if types(*said) != "moving,settled" || (*said)[1].Body.Outcome != "running" {
|
||||
t.Errorf("said %q, want settled with its outcome", types(*said))
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if types(*said) != "moving,settled" {
|
||||
t.Errorf("settled said twice: %q", types(*said))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleHeldOrFailedIsSaidStuckWithItsWords(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
b, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
|
||||
fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0})
|
||||
if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" || (*said)[1].Body.Words != "no space left" {
|
||||
t.Fatalf("said %v: %+v", types(*said), *said)
|
||||
}
|
||||
fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0})
|
||||
if types(*said) != "moving,stuck" {
|
||||
t.Errorf("stuck said twice: %q", types(*said))
|
||||
}
|
||||
// failed on the wanted fingerprint
|
||||
m2 := New()
|
||||
said2 := motionLog(m2)
|
||||
fill(m2, wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed))
|
||||
if types(*said2) != "moving,stuck" || (*said2)[1].Body.Outcome != "failed" {
|
||||
t.Errorf("failed: said %q", types(*said2))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWantTheWatchmanSaysStoppedMovingIsSaidStuckPastItsBound(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, "want.ace.web")
|
||||
v, err := delivery.EncodeStuck(delivery.Stuck{Bucket: delivery.AssignerWantsBucket, Key: "want.ace.web", Waited: "the node agent of ace to run web", Since: t0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fill(m, Entry{Bucket: delivery.StuckBucket, Key: sk, Value: v, At: t0})
|
||||
if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" {
|
||||
t.Errorf("said %q: %+v", types(*said), *said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatSettlesWithoutBeingSeenMovingSaysNothing(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if len(*said) != 0 {
|
||||
t.Errorf("said %v for a module that never moved", types(*said))
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing is said of a module in motion before the buckets have been read: a want read ahead of its have is not a node behind. What is
|
||||
// in motion when they have been read is said then, and a module that settles after is said settled.
|
||||
func TestMotionIsSaidOnlyOnceTheBucketsHaveBeenRead(t *testing.T) {
|
||||
m := New()
|
||||
var said []MotionEvent
|
||||
m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) })
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), wantE(t, "ace", "mail", delivery.Want{FP: fpNew}))
|
||||
m.Initial(delivery.AssignerWantsBucket)
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK))
|
||||
m.Initial(delivery.HaveBucket)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said %v before the stuck keys were read", types(said))
|
||||
}
|
||||
m.Initial(delivery.StuckBucket)
|
||||
if len(said) != 1 || said[0].Type != MotionMoving || said[0].Body.Module != "mail" {
|
||||
t.Fatalf("said %v: %+v, want only mail moving (web is already there)", types(said), said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "mail", fpNew, delivery.HealthOK))
|
||||
if types(said) != "moving,settled" {
|
||||
t.Errorf("said %q", types(said))
|
||||
}
|
||||
}
|
||||
|
||||
// assigner_wants lists a node's own want (want.<node>.node, ADR 0539 clause 6) among the wants, and never as unreadable:
|
||||
// on 2026-10-25 every node's own want was listed unreadable, though its value was what the assigner writes (issue 1945).
|
||||
func TestTheWantsListANodesOwnWant(t *testing.T) {
|
||||
own, err := delivery.WantNodeKey("ace")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeNodeWant(delivery.NodeWant{Composed: "sha256:" + strings.Repeat("d", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moduleKey, _ := delivery.WantKey("ace", "web")
|
||||
moduleValue, _ := delivery.EncodeWant(delivery.Want{FP: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
state := map[string][]byte{own: value, moduleKey: moduleValue}
|
||||
r := Reader{WantsState: StateRead{
|
||||
Keys: func() ([]string, error) { return []string{own, moduleKey}, nil },
|
||||
Get: func(k string) (json.RawMessage, error) { return state[k], nil },
|
||||
}}
|
||||
got, err := r.Wants("", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Unreadable) != 0 || got.Wants[own] == nil || got.Wants[moduleKey] == nil {
|
||||
t.Fatalf("the wants: %v, unreadable %v", got.Wants, got.Unreadable)
|
||||
}
|
||||
if got, _ := r.Wants("g14", ""); len(got.Wants) != 0 {
|
||||
t.Errorf("another node's wants: %v", got.Wants)
|
||||
}
|
||||
if got, _ := r.Wants("ace", "web"); got.Wants[own] != nil {
|
||||
t.Errorf("the node's own want was listed under the module web")
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,7 @@ type manifest struct {
|
||||
Reads []string `json:"reads"`
|
||||
Uses []string `json:"uses"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
@@ -83,17 +84,21 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
if m.Module != "mesh-delivery-view" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", got, want)
|
||||
}
|
||||
for _, k := range []string{"emits", "consumes", "provides", "claims"} {
|
||||
for _, k := range []string{"consumes", "provides", "claims"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the view writes only its answers", k)
|
||||
}
|
||||
}
|
||||
// The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves.
|
||||
if !reflect.DeepEqual(m.Emits, []string{"delivered", "moving", "settled", "stuck"}) {
|
||||
t.Errorf("emits = %v, want exactly delivered, moving, settled and stuck (1906/D9, D10)", m.Emits)
|
||||
}
|
||||
// The view uses no build seat: it reads their workers by the broker's named grant and publishes no ask (ADR 0525 §2).
|
||||
if len(m.Uses) != 0 {
|
||||
t.Errorf("uses = %v, want none", m.Uses)
|
||||
@@ -110,7 +115,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
}
|
||||
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
||||
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -9,7 +9,17 @@
|
||||
"indexer_candidate",
|
||||
"indexer_candidates",
|
||||
"builder_build",
|
||||
"builder_builds"
|
||||
"builder_builds",
|
||||
"assigner_wants",
|
||||
"assigner_want",
|
||||
"assigner_differences",
|
||||
"node_haves"
|
||||
],
|
||||
"emits": [
|
||||
"delivered",
|
||||
"moving",
|
||||
"settled",
|
||||
"stuck"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "answers", "history": 5 }
|
||||
@@ -17,10 +27,13 @@
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-delivery-assigner.asks",
|
||||
"mesh-delivery-assigner.differences",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-gate.asks",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -99,6 +100,8 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
|
||||
{Bucket: delivery.StuckBucket, History: 20},
|
||||
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
|
||||
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
|
||||
{Bucket: delivery.AssignerWantsBucket, History: 20},
|
||||
{Bucket: delivery.HaveBucket, History: 20},
|
||||
} {
|
||||
kv, err := js.CreateKeyValue(ctx, c)
|
||||
if err != nil {
|
||||
@@ -265,6 +268,8 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
|
||||
{Bucket: delivery.StuckBucket, History: 20},
|
||||
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
|
||||
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
|
||||
{Bucket: delivery.AssignerWantsBucket, History: 20},
|
||||
{Bucket: delivery.HaveBucket, History: 20},
|
||||
} {
|
||||
kv, err := js.CreateKeyValue(ctx, c)
|
||||
if err != nil {
|
||||
@@ -302,3 +307,134 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
|
||||
}
|
||||
t.Fatalf("no pulse for the ask running before the start (%s)", pk)
|
||||
}
|
||||
|
||||
// TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt (1906/D6, ADR 0539 §19): started whole, the watchman mirrors a want of
|
||||
// the assigner as a pulse owed by the node's agent, one that stood before its start as well, and the node-engine's have that
|
||||
// names the want's fingerprint deletes the pulse and writes no stuck key.
|
||||
func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "w")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
srv, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ctl.Close)
|
||||
js, err := jetstream.New(ctl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, c := range []jetstream.KeyValueConfig{
|
||||
{Bucket: delivery.PresenceBucket, History: 20},
|
||||
{Bucket: delivery.StuckBucket, History: 20},
|
||||
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
|
||||
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
|
||||
{Bucket: delivery.AssignerWantsBucket, History: 20},
|
||||
{Bucket: delivery.HaveBucket, History: 20},
|
||||
} {
|
||||
kv, err := js.CreateKeyValue(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[c.Bucket] = kv
|
||||
}
|
||||
fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
putWant := func(node, module, fp string) {
|
||||
t.Helper()
|
||||
body, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, key, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
putHave := func(node, module, fp string) {
|
||||
t.Helper()
|
||||
body, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveKey(node, module)
|
||||
if _, err := kvs[delivery.HaveBucket].Put(ctx, key, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
putHave("n2", "web", fpOld)
|
||||
putWant("n2", "web", fpNew) // stood before the start
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-watchman", User: "n1.mesh-delivery-watchman", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
pulse := func(node, module string) string {
|
||||
wk, _ := delivery.WantKey(node, module)
|
||||
pk, err := delivery.PulseKey(delivery.AssignerWantsBucket, wk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return pk
|
||||
}
|
||||
until := func(what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("never saw: %s", what)
|
||||
}
|
||||
has := func(key string) bool { _, err := kvs[delivery.PulseBucket].Get(ctx, key); return err == nil }
|
||||
until("a pulse for the want that stood before the start", func() bool { return has(pulse("n2", "web")) })
|
||||
putWant("n2", "mail", fpNew)
|
||||
until("a pulse for a want written after the start", func() bool { return has(pulse("n2", "mail")) })
|
||||
e, err := kvs[delivery.PulseBucket].Get(ctx, pulse("n2", "web"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, err := delivery.DecodePulse(e.Value()); err != nil || !strings.Contains(p.Owes, "n2") {
|
||||
t.Errorf("the pulse = %+v, %v; want it owed by the agent of n2", p, err)
|
||||
}
|
||||
putHave("n2", "web", fpNew)
|
||||
until("the pulse of the want its have names deleted", func() bool { return !has(pulse("n2", "web")) })
|
||||
if !has(pulse("n2", "mail")) {
|
||||
t.Errorf("the pulse of another want ended with web's have")
|
||||
}
|
||||
for _, m := range []string{"web"} {
|
||||
wk, _ := delivery.WantKey("n2", m)
|
||||
sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, wk)
|
||||
if _, err := kvs[delivery.StuckBucket].Get(ctx, sk); err == nil {
|
||||
t.Errorf("a stuck key was written for the want of %s that was applied", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,11 +229,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
var kvs struct{ presence, pulse, stuck jetstream.KeyValue }
|
||||
var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue }
|
||||
for _, b := range []struct {
|
||||
name string
|
||||
into *jetstream.KeyValue
|
||||
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} {
|
||||
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck},
|
||||
{delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} {
|
||||
if *b.into, err = bucket(b.name); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -280,6 +281,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent
|
||||
// until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse.
|
||||
haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
|
||||
// keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same.
|
||||
taken := map[string]jetstream.KeyWatcher{}
|
||||
@@ -337,6 +348,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
go func() {
|
||||
errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) })
|
||||
}()
|
||||
go func() {
|
||||
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
|
||||
}()
|
||||
go func() {
|
||||
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
|
||||
}()
|
||||
for lang, sw := range taken {
|
||||
lang, sw := lang, sw
|
||||
go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }()
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -52,3 +55,96 @@ func (w *Watchman) Step(language, key string, value []byte, op Op) {
|
||||
}
|
||||
w.mirrorWithin(w.stepBound, bucket, id, describe("the ask %s to end on %s", id, node), node, since, 0)
|
||||
}
|
||||
|
||||
// A want is a move too (novox/hq ADR 0539 §19, issue 1906): the assigner writes `want.<node>.<module>` in its wants bucket,
|
||||
// and the node's agent owes the node-engine's have naming the want's fingerprint. The pulse is the pulse of that bucket under
|
||||
// the want's key, owed by the node's agent, and it begins when the assigner wrote the want. The have naming the fingerprint
|
||||
// ends it; a want that is removed, or a key deleted, ends it as well. Nothing renews it: the bound is how long a node may
|
||||
// take to apply a want, the same as a build step's (StepBound). Keys of that bucket that are not a want of a module (a hold,
|
||||
// and the node's own key) are left.
|
||||
|
||||
// Want is a change of a key of the assigner's wants bucket, read from the watch (at the start the keys that stand, then each
|
||||
// change), at the time the entry was written.
|
||||
func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
|
||||
node, module, err := delivery.ParseWantKey(key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch op {
|
||||
case Expired:
|
||||
return
|
||||
case Delete:
|
||||
delete(w.wants, key)
|
||||
w.moved(delivery.AssignerWantsBucket, key)
|
||||
return
|
||||
}
|
||||
want, err := delivery.DecodeWant(value)
|
||||
if err != nil {
|
||||
w.say("the want %s cannot be read, and is left: %v", key, err)
|
||||
return
|
||||
}
|
||||
if want.Removed {
|
||||
delete(w.wants, key)
|
||||
w.moved(delivery.AssignerWantsBucket, key)
|
||||
return
|
||||
}
|
||||
w.wants[key] = want.FP
|
||||
if w.have(node, module) == want.FP {
|
||||
w.moved(delivery.AssignerWantsBucket, key)
|
||||
return
|
||||
}
|
||||
if at.IsZero() {
|
||||
at = w.now()
|
||||
}
|
||||
w.mirrorWithin(w.stepBound, delivery.AssignerWantsBucket, key,
|
||||
describe("the node agent of %s to run %s at %s", node, module, shortFP(want.FP)), want.FP, at, 0)
|
||||
}
|
||||
|
||||
// Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the
|
||||
// fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left.
|
||||
func (w *Watchman) Have(key string, value []byte, op Op) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch op {
|
||||
case Expired:
|
||||
return
|
||||
case Delete:
|
||||
delete(w.haves, key)
|
||||
return
|
||||
}
|
||||
have, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
w.say("the have %s cannot be read, and is left: %v", key, err)
|
||||
return
|
||||
}
|
||||
w.haves[key] = have.FP
|
||||
wk, err := delivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if fp, ok := w.wants[wk]; ok && fp == have.FP {
|
||||
w.moved(delivery.AssignerWantsBucket, wk)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watchman) have(node, module string) string {
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return w.haves[key]
|
||||
}
|
||||
|
||||
func shortFP(fp string) string {
|
||||
hex := strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
if len(hex) > 12 {
|
||||
hex = hex[:12]
|
||||
}
|
||||
return hex
|
||||
}
|
||||
|
||||
@@ -57,6 +57,8 @@ type Watchman struct {
|
||||
presence map[string]presence
|
||||
pulses map[string]*pulse // live pulses, by pulse key
|
||||
stuck map[string]delivery.Stuck
|
||||
wants map[string]string // the fingerprint of each want that stands, by want key
|
||||
haves map[string]string // the fingerprint each node runs of a module, by have key
|
||||
bound time.Duration
|
||||
// stepBound is how long a build step may run once a node took the ask; a test shortens it.
|
||||
stepBound time.Duration
|
||||
@@ -65,7 +67,7 @@ type Watchman struct {
|
||||
// New is a watchman writing to out, reading the time from now.
|
||||
func New(out Out, now func() time.Time) *Watchman {
|
||||
return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{},
|
||||
pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, bound: delivery.PulseMoveBound, stepBound: StepBound}
|
||||
pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound}
|
||||
}
|
||||
|
||||
// Logf sets where a write that failed is said; it is silent until set.
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -531,3 +532,91 @@ func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908
|
||||
t.Errorf("stuck %v pulses %v", o2.stuck, o2.pulses)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D6 (ADR 0539 §19): a want is a pulse owed by the node's agent from when the assigner wrote it, until the have names
|
||||
// its fingerprint.
|
||||
func wantValue(t *testing.T, fp string, removed bool) []byte {
|
||||
t.Helper()
|
||||
w := delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed}
|
||||
if removed {
|
||||
w.Composed = ""
|
||||
}
|
||||
b, err := delivery.EncodeWant(w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func haveValue(t *testing.T, fp string) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var (
|
||||
fpOld = delivery.SourcePrefix2 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
fpNew = delivery.SourcePrefix2 + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
)
|
||||
|
||||
func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
c.t = t0.Add(10 * time.Second)
|
||||
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")
|
||||
p, ok := o.pulses[pk]
|
||||
if !ok || p.Key != "want.ace.web" || !strings.Contains(p.Owes, "ace") || !strings.Contains(p.Owes, "web") {
|
||||
t.Fatalf("pulses = %v", o.pulses)
|
||||
}
|
||||
w.Have("ace.web", haveValue(t, fpOld), Put) // the node still runs the old one
|
||||
if _, ok := o.pulses[pk]; !ok {
|
||||
t.Fatalf("a have of another fingerprint ended the pulse")
|
||||
}
|
||||
w.Have("ace.web", haveValue(t, fpNew), Put)
|
||||
if _, ok := o.pulses[pk]; ok {
|
||||
t.Errorf("the have naming the want's fingerprint left the pulse standing: %v", o.pulses)
|
||||
}
|
||||
if len(o.stuck) != 0 {
|
||||
t.Errorf("stuck = %v", o.stuck)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHaveThatCameBeforeTheWantEndsItAtOnceAndARemovedWantHasNoPulse(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
c.t = t0.Add(time.Second)
|
||||
w.Have("ace.web", haveValue(t, fpNew), Put)
|
||||
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
w.Want("want.ace.mail", wantValue(t, "", true), Put, t0)
|
||||
if len(o.pulses) != 0 {
|
||||
t.Errorf("pulses = %v", o.pulses)
|
||||
}
|
||||
// the want moving on to a new fingerprint makes a new pulse; the key `want.ace.node` and a held key are left
|
||||
w.Want("want.ace.web", wantValue(t, fpOld, false), Put, t0.Add(time.Second))
|
||||
w.Want("want.ace.node", []byte(`{}`), Put, t0)
|
||||
w.Want("held.web.bbbb", []byte(`{}`), Put, t0)
|
||||
if len(o.pulses) != 1 {
|
||||
t.Errorf("pulses = %v, want the web want's only", o.pulses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) {
|
||||
o, c := newOut(), &clock{t: t0}
|
||||
w := New(o, c.now)
|
||||
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")
|
||||
c.t = t0.Add(31 * time.Minute)
|
||||
w.Pulse(pk, Expired, c.t)
|
||||
if s, ok := o.stuck[stuckKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")]; !ok || s.Moved || !s.Since.Equal(t0) {
|
||||
t.Fatalf("stuck = %v", o.stuck)
|
||||
}
|
||||
w2, o2, c2 := setup()
|
||||
c2.t = t0.Add(time.Second)
|
||||
w2.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
w2.Want("want.ace.web", nil, Delete, t0)
|
||||
if len(o2.pulses) != 0 {
|
||||
t.Errorf("a deleted want left its pulse: %v", o2.pulses)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps"}) {
|
||||
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-delivery-assigner.wants", "node-engine.have"}) {
|
||||
t.Errorf("reads = %v", reads)
|
||||
}
|
||||
for _, k := range []string{"emits", "context", "items", "consumes", "provides", "claims"} {
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.steps"
|
||||
"mesh-build-node.steps",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"node-engine.have"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "presence", "history": 20 },
|
||||
|
||||
Reference in New Issue
Block a user