diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index d017c8d..7891870 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "fmt" "io" "os" "path/filepath" @@ -17,9 +18,12 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/provider" + + "mesh-delivery-assigner/internal/assign" ) // bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events @@ -73,6 +77,10 @@ func startBed(t *testing.T, leaveOut string) *bed { if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { t.Fatal(err) } + // The operator-channel seat's stream of conditions, which the assigner says its conditions on. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}}); err != nil { + t.Fatal(err) + } // The Go build seat's work queue, as the broker derives it from the seat's accepted kinds (delivery.AskStream). seat, _ := delivery.AskStream(delivery.AskGo) if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: seat, Subjects: []string{"mesh.seat.node-build-go.accept.>"}, @@ -85,6 +93,14 @@ func startBed(t *testing.T, leaveOut string) *bed { // start runs the program as the node does, through run with its environment and a broker file; it answers the // channel run's error arrives on. func (b *bed) start(t *testing.T) <-chan error { + t.Helper() + done, _ := b.startStoppable(t) + return done +} + +// startStoppable is start, and answers the function that stops this one program (a replaced assigner is stopped, then +// another started). +func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) { t.Helper() broker := filepath.Join(b.dir, "broker") cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"}) @@ -94,15 +110,19 @@ func (b *bed) start(t *testing.T) <-chan error { env := map[string]string{"MESH_BROKER_FILE": broker} ctx, stop := context.WithCancel(b.ctx) done := make(chan error, 1) - go func() { done <- run(ctx, func(k string) string { return env[k] }, io.Discard) }() - t.Cleanup(func() { + go func() { + done <- run(ctx, func(k string) string { return env[k] }, io.Discard) + close(done) // a program that ended on its own (a refused start) is not waited for again at the cleanup + }() + stopped := func() { stop() select { case <-done: case <-time.After(10 * time.Second): } - }) - return done + } + t.Cleanup(stopped) + return done, stopped } // 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable, @@ -159,6 +179,27 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) { } } +// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the +// move to it, in mesh-controller_composed. +func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string { + t.Helper() + key, err := delivery.ComposedKey(node, module, fp) + if err != nil { + t.Fatal(err) + } + digest := "sha256:" + strings.Repeat(string(d), 64) + cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement} + if judgement != delivery.JudgedNeither { + cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told" + } + v, err := delivery.EncodeComposedDigest(cd) + if err != nil { + t.Fatal(err) + } + b.putJSON(t, delivery.ControllerComposedBucket, key, v) + return digest +} + // manifest keeps a manifest in the indexer's bucket by its digest and answers the digest. func (b *bed) manifest(t *testing.T, module, body string) string { t.Helper() @@ -240,10 +281,10 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte { return nil } -// 1297/B4, B5 on the bus: the controller sent F to g14 while the canary ace does not run F healthy yet. Started whole, -// the assigner wants F on the canary, writes the difference on g14 explained by rule 3, and on ace one unexplained (the -// controller sends ace the older fingerprint while ace wants F). -func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) { +// 1297/B4 on the bus, 1906/D5: the controller sends ace and g14 builds while the canary ace does not run F healthy yet. +// Started whole, the assigner wants F on the canary, holds g14 back with no want, and writes no difference at all: there +// is no send to compare with (ADR 0539 §8). +func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { b := startBed(t, "") fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64) digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) @@ -258,6 +299,8 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) { gkey, _ := delivery.ControllerGrantKey("web", fNew) grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{}) b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant) + b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither) + b.compose(t, "g14", "web", fNew, 'c', delivery.JudgedNeither) for _, node := range []string{"ace", "g14"} { on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) b.putJSON(t, delivery.PresenceBucket, node, on) @@ -266,8 +309,6 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) { } b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`)) b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`)) - b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-8"), []byte(`{"id":"b-8","repository":"novox/web","source-fingerprint":"`+fOld+`"}`)) - b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-9"), []byte(`{"id":"b-9","repository":"novox/web","source-fingerprint":"`+fNew+`"}`)) b.start(t) wkey, _ := delivery.WantKey("ace", "web") @@ -275,15 +316,9 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) { if err != nil || want.FP != fNew { t.Fatalf("the canary ace's want = %+v, %v", want, err) } - dkey, _ := delivery.DifferenceKey("g14", "web") - d, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, dkey)) - if err != nil || d.Rule != delivery.RuleCanary || d.Sent != fNew { - t.Fatalf("g14's difference = %+v, %v; want rule 3", d, err) - } - akey, _ := delivery.DifferenceKey("ace", "web") - a, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, akey)) - if err != nil || !a.Unexplained || a.Want != fNew || a.Sent != fOld { - t.Fatalf("ace's difference = %+v, %v; want it unexplained", a, err) + time.Sleep(500 * time.Millisecond) + if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 { + t.Errorf("differences were written: %v", keys) } if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil { t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value()) @@ -315,3 +350,426 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) { t.Fatalf("the ask = %+v, %v", a, err) } } + +// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key. +func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition { + t.Helper() + stream, err := b.js.Stream(b.ctx, "CONDITIONS") + if err != nil { + t.Fatal(err) + } + out := map[string]asks.Condition{} + cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy}) + if err != nil { + t.Fatal(err) + } + for { + batch, err := cons.FetchNoWait(100) + if err != nil { + t.Fatal(err) + } + n := 0 + for m := range batch.Messages() { + n++ + c, err := asks.DecodeCondition(m.Data()) + if err != nil { + t.Fatal(err) + } + out[c.Key] = c + } + if n == 0 { + return out + } + } +} + +// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the +// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node +// and the check's words. +func TestAFailedDataModuleIsAskedToRestore(t *testing.T) { + b := startBed(t, "") + fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64) + digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`) + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + b.putJSON(t, delivery.ModulesBucket, "mail", mod) + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + b.putJSON(t, delivery.PresenceBucket, "ace", on) + failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}) + b.putJSON(t, delivery.HaveBucket, "ace.mail", failed) + wantKey, _ := delivery.WantKey("ace", "mail") + want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)}) + b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want) + b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`)) + b.start(t) + + akey, _ := delivery.RestoreAskKey("ace", "mail", fNew) + a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey)) + if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" { + t.Fatalf("the restore ask = %+v, %v", a, err) + } + if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew { + t.Errorf("the data module was set back: want = %+v, %v", w, err) + } + key := "" + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) { + for k, c := range b.conditionsSaid(t) { + if c.State == asks.StateRaised && c.Subject.ID == "mail" { + if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") { + t.Fatalf("the condition = %+v, want it to name ace and the check's words", c) + } + key = k + } + } + } + if key == "" { + t.Fatal("no condition for the held fingerprint was said") + } +} + +// 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants +// of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost +// and nothing written again. +func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) { + b := startBed(t, "") + const modules = 40 + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + b.putJSON(t, delivery.PresenceBucket, "ace", on) + names := make([]string, modules) + for i := range names { + name := fmt.Sprintf("mod%02d", i) + names[i] = name + fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16) + digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name)) + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + b.putJSON(t, delivery.ModulesBucket, name, mod) + built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) + if err != nil { + t.Fatal(err) + } + bkey, _ := delivery.BuilderBuildKey(name, fp) + b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built) + gkey, _ := delivery.ControllerGrantKey(name, fp) + grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{}) + b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant) + b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither) + b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i))) + } + count := func() int { + keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx) + if err != nil { + return 0 + } + return len(keys) + } + _, stop := b.startStoppable(t) + for deadline := time.Now().Add(20 * time.Second); count() == 0 && time.Now().Before(deadline); time.Sleep(time.Millisecond) { + } + stop() // replaced, wherever its pass had got to + b.start(t) + for deadline := time.Now().Add(30 * time.Second); count() < modules && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + } + time.Sleep(time.Second) // and no pass of the second writes any again + for _, name := range names { + key, _ := delivery.WantKey("ace", name) + history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key) + if err != nil { + t.Errorf("the want of %s was never written: %v", name, err) + continue + } + if len(history) != 1 { + t.Errorf("the want of %s was written %d times, want once", name, len(history)) + } + } +} + +// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it +// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want +// `want..node` is written once the node's composition exists, and again when its digest changes. +func TestAWantWaitsForItsComposition(t *testing.T) { + b := startBed(t, "") + fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64) + digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + b.putJSON(t, delivery.ModulesBucket, "web", mod) + built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) + bkey, _ := delivery.BuilderBuildKey("web", fNew) + b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built) + gkey, _ := delivery.ControllerGrantKey("web", fNew) + grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{}) + b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant) + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + b.putJSON(t, delivery.PresenceBucket, "ace", on) + b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`)) + b.start(t) + wkey, _ := delivery.WantKey("ace", "web") + time.Sleep(time.Second) + if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil { + t.Fatalf("a want was written before the composition existed: %s", e.Value()) + } + first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither) + if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first { + t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first) + } + second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither) + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil { + if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second { + break + } + } + } + if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second { + t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second) + } + // the node's own want + nkey, _ := delivery.WantNodeKey("ace") + if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil { + t.Fatalf("the node's own want was written before its composition: %s", e.Value()) + } + ownKey, _ := delivery.ComposedNodeKey("ace") + own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither}) + b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own) + if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) { + t.Fatalf("the node's own want = %+v, %v", w, err) + } +} + +// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned. +func (b *bed) fixture(t *testing.T, name, fp, manifest string) { + t.Helper() + digest := b.manifest(t, name, manifest) + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + b.putJSON(t, delivery.ModulesBucket, name, mod) + built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) + bkey, _ := delivery.BuilderBuildKey(name, fp) + b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built) + gkey, _ := delivery.ControllerGrantKey(name, fp) + grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{}) + b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant) + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + b.putJSON(t, delivery.PresenceBucket, "ace", on) + b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`)) +} + +func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) { + t.Helper() + key, _ := delivery.WantKey(node, module) + v, err := delivery.EncodeWant(w) + if err != nil { + t.Fatal(err) + } + b.putJSON(t, delivery.AssignerWantsBucket, key, v) +} + +// wantNow waits until the want of module on node satisfies ok and answers it. +func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.Want) bool) delivery.Want { + t.Helper() + key, _ := delivery.WantKey(node, module) + var last delivery.Want + for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil { + if w, err := delivery.DecodeWant(e.Value()); err == nil { + last = w + if ok(w) { + return w + } + } + } + } + t.Fatalf("never saw: %s; the want is %+v", what, last) + return last +} + +// answer says the teller wrote the operator's answer to the ask at key. +func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") } + +// answerTo is answer to the ask with this id (the router's warrant names it). +func (b *bed) answerTo(t *testing.T, key, option, askID string) { + t.Helper() + v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: askID + "@2026-10-24T12:00:00Z", Words: "the operator"}) + if err != nil { + t.Fatal(err) + } + b.putJSON(t, delivery.AnswersBucket, key, v) +} + +func (b *bed) hasKey(bucket, key string) bool { + _, err := b.buckets[bucket].Get(b.ctx, key) + return err == nil +} + +// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules, +// and clears the condition. +func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) { + b := startBed(t, "") + fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64) + b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`) + b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither) + b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither) + healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before + failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}) + b.putJSON(t, delivery.HaveBucket, "ace.web", failed) + b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)}) + b.start(t) + w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld }) + if w.Newest != fNew { + t.Fatalf("the set-back want = %+v", w) + } + hold, _ := delivery.HoldAskKey("web", fNew) + b.get(t, delivery.AssignerAsksBucket, hold) + b.answer(t, hold, "release") + w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew }) + if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 { + t.Errorf("the want = %+v", w) + } + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + cleared := false + for _, c := range b.conditionsSaid(t) { + cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared) + } + if cleared { + return + } + } + t.Error("the condition of the held fingerprint was not cleared") +} + +// askNow waits for the backup ask at key to be the one numbered asking and answers it. +func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk { + t.Helper() + for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil { + if a, err := delivery.DecodeAssignerAsk(key, e.Value()); err == nil && a.Asking == asking { + return a + } + } + } + t.Fatalf("the ask at %s never became asking %d", key, asking) + return delivery.AssignerAsk{} +} + +// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer +// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want +// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint. +func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) { + b := startBed(t, "") + fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64) + b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`) + first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither) + b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first}) + held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"}) + b.putJSON(t, delivery.HaveBucket, "ace.web", held) + b.start(t) + akey, _ := delivery.BackupAskKey("ace", "web", fNew) + a := b.askNow(t, akey, 0) + if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) { + t.Fatalf("the backup ask = %+v", a) + } + // an answer to another ask is no answer to this one + b.answerTo(t, akey, delivery.BackupSkip, "someone-else") + time.Sleep(time.Second) + if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" { + t.Fatalf("an answer to another ask was taken: %+v", w) + } + b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID) + w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain }) + if w1.FP != fNew || w1.Composed != first { + t.Errorf("the want = %+v", w1) + } + // the node retries and the backup fails again: a new have value, asked again under the next number + again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"}) + b.putJSON(t, delivery.HaveBucket, "ace.web", again) + a2 := b.askNow(t, akey, 2) + if a2.Ask.ID == a.Ask.ID { + t.Fatalf("asked again under the same id") + } + b.answerTo(t, akey, delivery.BackupAgain, a2.Ask.ID) + w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain && w.Words != w1.Words }) + if w2.Words == w1.Words { + t.Errorf("the second answer left the want as it was: %+v", w2) + } + // a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it + second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither) + b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second}) + a3 := b.askNow(t, akey, 3) + if !strings.Contains(a3.Ask.Explanation, second) { + t.Errorf("the ask for the new revision = %+v", a3) + } + time.Sleep(time.Second) + if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" { + t.Errorf("the old answer was taken for the new revision: %+v", w) + } +} + +const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}` + +// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate +// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged +// neither is written at once. +func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) { + b := startBed(t, "") + fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64) + b.fixture(t, "web", fNew, recorded) + old := "sha256:" + strings.Repeat("a", 64) + b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old}) + nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates) + b.start(t) + akey, _ := delivery.RecordAskKey("ace", "web", nw) + a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey)) + if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") { + t.Fatalf("the record ask = %+v, %v", a, err) + } + time.Sleep(time.Second) + if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld { + t.Fatalf("the move was written before the yes: %+v", w) + } + b.answer(t, akey, assign.OptionYes) + b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw }) + // a new revision at the same fingerprint, judged to move data, waits as well + rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData) + rkey, _ := delivery.RecordAskKey("ace", "web", rev) + b.get(t, delivery.AssignerAsksBucket, rkey) + time.Sleep(time.Second) + if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw { + t.Fatalf("the revision was written before the yes: %+v", w) + } + // and a revision judged neither is written at once + free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither) + b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free }) +} + +// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the +// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged +// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made. +func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) { + b := startBed(t, "") + fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64) + b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`) + running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + b.putJSON(t, delivery.HaveBucket, "ace.bus", running) + b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)}) + b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither) + b.start(t) + time.Sleep(time.Second) + if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld { + t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w) + } + // the controller composes the bus at the fingerprint it runs with a new user list + key, _ := delivery.ComposedKey("ace", "bus", fOld) + free := "sha256:" + strings.Repeat("e", 64) + v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64), + Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"}) + if err != nil { + t.Fatal(err) + } + b.putJSON(t, delivery.ControllerComposedBucket, key, v) + w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free }) + if w.FP != fOld { + t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w) + } + rkey, _ := delivery.RecordAskKey("ace", "bus", free) + if b.hasKey(delivery.AssignerAsksBucket, rkey) { + t.Errorf("the user-list revision was asked") + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 0ca3642..54a5dc1 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -21,6 +21,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "git.novox.be/novox/mesh-sdk/go/process" @@ -42,6 +43,8 @@ var readBuckets = []string{ provider.AssignmentsBucket, delivery.BuildsBucket, delivery.ControllerGrantsBucket, + delivery.ControllerComposedBucket, + delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.CandidatesBucket, delivery.ManifestsBucket, @@ -130,7 +133,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err) } logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs)) - return follow(ctx, js, kvs, logger) + return follow(ctx, js, kvs, sayVia(ctx, js), logger) } // watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and @@ -138,6 +141,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro var watched = []string{ provider.AssignmentsBucket, delivery.ControllerGrantsBucket, + delivery.ControllerComposedBucket, + delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.CandidatesBucket, delivery.ModulesBucket, @@ -151,7 +156,7 @@ var watched = []string{ // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after // each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. -func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *log.Logger) error { +func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error { w := newWorld() dirty := make(chan struct{}, 1) wake := func() { @@ -215,7 +220,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *lo case <-settled: } } - p := &passer{kvs: kvs, w: w, logf: logger.Printf} + p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}} p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf) wake() for { @@ -230,3 +235,20 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *lo } } } + +// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name +// (ADR 0437 clause 9). +func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error { + return func(c asks.Condition) error { + subject, err := asks.ConditionSubject(delivery.AssignerModule) + if err != nil { + return err + } + data, err := asks.EncodeCondition(c) + if err != nil { + return err + } + _, err = js.Publish(ctx, subject, data) + return err + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index c5855fd..c736c38 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -3,8 +3,10 @@ package main import ( "context" "errors" + "strings" "time" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/sdkversion" @@ -19,76 +21,43 @@ type passer struct { w *world logf func(string, ...any) sdk string - asks *composeAsks + // say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a + // pass says a condition once and a restart says it again (the router keeps one per key). + say func(asks.Condition) error + said map[string]bool + // answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want. + answeredAt map[string]uint64 + asks *composeAsks } func (p *passer) pass(ctx context.Context) { p.compositions(ctx) - rules := p.wants(ctx) - p.differences(ctx, rules) + p.wants(ctx) + p.nodeWants(ctx) + p.backups(ctx) } -// differences writes, for each node and module the assigner wants or the controller sends, the difference between the -// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that -// ended; its history keeps what it was. -func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) { - _, assignments := p.w.assigned() - wanted := p.w.wantsOf() - trunk := p.w.trunk() - pairs := map[[2]string]bool{} - for _, a := range assignments { - pairs[[2]string{a.Node, a.Module}] = true - } - for module, byNode := range wanted { - for node := range byNode { - pairs[[2]string{node, module}] = true - } - } - written := p.w.read(delivery.AssignerDifferencesBucket) - keep := map[string]bool{} - for pair := range pairs { - node, module := pair[0], pair[1] - key, err := delivery.DifferenceKey(node, module) +// nodeWants writes each node's own want, `want..node`, naming the digest of the node's own composition, once the +// controller has composed it and again whenever that digest changes (ADR 0539 §6). +func (p *passer) nodeWants(ctx context.Context) { + written := p.w.read(delivery.AssignerWantsBucket) + for node, d := range p.w.nodeComposed() { + key, err := delivery.WantNodeKey(node) if err != nil { continue } - var want *delivery.Want - if w, ok := wanted[module][node]; ok { - want = &w - } - var sent assign.Sent - if a, ok := assignments[node+"."+module]; ok { - sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)} - } - d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node]) - if !ok { + if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest { continue } - keep[key] = true - if before, err := delivery.DecodeDifference(written[key]); err == nil { - before.V = 0 - if delivery.SameDifference(before, d) { - continue - } - } - p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) }) - } - for key := range written { - if keep[key] { - continue - } - if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil { - p.logf("the difference %s that ended could not be deleted: %v", key, err) - continue - } - p.w.put(delivery.AssignerDifferencesBucket, key, nil) + want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"} + p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) }) } } // wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and -// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node -// back, for the comparison. -func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding { +// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller +// sends: the differences bucket is kept read-only with its history (ADR 0539 §8). +func (p *passer) wants(ctx context.Context) { get := kvGetter(p.kvs[delivery.ManifestsBucket]) assigned, _ := p.w.assigned() trunk := p.w.trunk() @@ -109,7 +78,6 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding } } wanted := p.w.wantsOf() - rules := map[string]map[string]assign.Holding{} names := map[string]bool{} for name := range trunk { names[name] = true @@ -119,18 +87,32 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding } for name := range names { fp := trunk[name].FP2 - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]} + name := name + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], + Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) }, + Restored: func(node string) bool { + key, err := delivery.RestoreAskKey(node, name, fp) + return err == nil && p.answeredWith(key, assign.OptionRestore) + }, + Allowed: func(node, digest string) bool { + key, err := delivery.RecordAskKey(node, name, digest) + return err == nil && p.answeredWith(key, assign.OptionYes) + }} if fp != "" { held, _ := delivery.HeldKey(name, fp) grant, _ := delivery.ControllerGrantKey(name, fp) - m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant) + hold, _ := delivery.HoldAskKey(name, fp) + m.Released = p.answeredWith(hold, "release") + m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant) + if m.Released { + p.clearHeld(name, fp) + } } last := map[string]string{} for _, node := range m.Nodes { last[node] = p.w.lastHealthy(node, name, fp) } out := assign.Wants(m, nodes, wanted[name], last) - rules[name] = out.Rules if out.Held != nil { key, err := delivery.HeldKey(name, fp) if err == nil { @@ -140,6 +122,13 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding } if out.Held != nil || m.Held { p.holdAsk(ctx, name, fp) + p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0) + } + for _, node := range out.Restore { + p.restoreAsk(ctx, name, node, fp) + } + for _, need := range out.Record { + p.recordAsk(ctx, name, fp, need) } for node, want := range out.Wants { key, err := delivery.WantKey(node, name) @@ -151,7 +140,6 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) }) } } - return rules } // compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the @@ -230,3 +218,152 @@ func (p *passer) holdAsk(ctx context.Context, module, fp string) { a := assign.HoldAsk(key, module, fp, time.Now()) p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) }) } + +// heldHold is the hold of fp as written: the node whose health check failed and its words. +func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held { + if fresh != nil { + return *fresh + } + key, err := delivery.HeldKey(module, fp) + if err != nil { + return delivery.Held{} + } + h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key]) + return h +} + +// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the +// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask. +func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *delivery.Held, data bool) { + h := p.heldHold(module, fp, fresh) + c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data) + if p.say == nil || p.said[c.Key+".raised"] { + return + } + if err := p.say(c); err != nil { + p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err) + return + } + p.said[c.Key+".raised"] = true +} + +// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node. +func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) { + key, err := delivery.RestoreAskKey(node, module, fp) + if err != nil || p.w.has(delivery.AssignerAsksBucket, key) { + return + } + words := p.heldHold(module, fp, nil).Words + a := assign.RestoreAsk(key, node, module, fp, words, time.Now()) + p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) }) +} + +// answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the +// operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19). +func (p *passer) answeredWith(key, option string) bool { + a, ok := p.w.answer(key) + return ok && a.Option == option +} + +// clearHeld clears, once, the condition of a hold the operator released. +func (p *passer) clearHeld(module, fp string) { + c := assign.HeldCondition(module, fp, "", "", false, false) + if p.say == nil || p.said[c.Key+".cleared"] { + return + } + if err := p.say(c); err != nil { + p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err) + return + } + p.said[c.Key+".cleared"] = true +} + +// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes. +func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) { + key, err := delivery.RecordAskKey(need.Node, module, need.Digest) + if err != nil || p.w.has(delivery.AssignerAsksBucket, key) { + return + } + a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now()) + p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) }) +} + +// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16): +// it asks whether to back up again or go on without a backup, raises the condition naming node, module and the backup's words, +// and writes the answer into the node's want for the module, which the node agent reads from it and from nothing else. +// +// The ask is bound to the composition (it names the digest, and a new revision at the same fingerprint is asked again under a +// new number) and an answer counts only if the router's warrant names the id of the ask now at the key (the warrant's first part, +// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again +// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen +// in a new have value after the answer was written. +func (p *passer) backups(ctx context.Context) { + wanted := p.w.wantsOf() + for key, value := range p.w.read(delivery.HaveBucket) { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + continue + } + have, err := delivery.DecodeHave(value) + if err != nil || have.Phase != delivery.PhaseHeld { + continue + } + want, ok := wanted[module][node] + if !ok || want.Removed { + continue + } + akey, err := delivery.BackupAskKey(node, module, want.FP) + if err != nil { + continue + } + current, there := delivery.DecodeAssignerAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey]) + if there != nil || assign.BackupAskDigest(current) != want.Composed { + after := -1 + if there == nil { + after = current.Asking + } + p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(after)) + continue + } + answer, answered := p.w.answer(akey) + if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != current.Ask.ID || + (answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) { + continue // not answered yet: the ask stands + } + words := assign.BackupAnswerWords(answer.Option, current.Asking) + haveChanges := p.w.changed(delivery.HaveBucket, key) + if want.Backup != answer.Option || want.Words != words { + want.Backup, want.Words = answer.Option, words + if wkey, err := delivery.WantKey(node, module); err == nil { + w := want + p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) }) + p.answeredAt[current.Ask.ID] = haveChanges + } + continue + } + if answer.Option != delivery.BackupAgain { + continue + } + // The answer is in the want. The node retried and the backup failed again when the have took another value since. + if seen, known := p.answeredAt[current.Ask.ID]; !known { + p.answeredAt[current.Ask.ID] = haveChanges // after a restart: wait for the next failure + } else if haveChanges > seen { + p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(current.Asking)) + } + } +} + +// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once. +func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) { + a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now()) + p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) }) + c := assign.BackupCondition(node, module, want.FP, words, true) + if p.say == nil || p.said[c.Key] { + return + } + if err := p.say(c); err != nil { + p.logf("the condition of the failed backup of %s on %s could not be said: %v", module, node, err) + return + } + p.said[c.Key] = true +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index fb49934..e5088ad 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -44,8 +44,8 @@ type world struct { manifests map[string]parsed // healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to. healthy map[string][]string - // sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one. - sent map[string]string + // changes counts, by bucket and key, the values this process has seen the key take: a value written again is another. + changes map[string]uint64 } // parsed is a manifest read by digest: its module and its manifest, or why it cannot be read. @@ -56,7 +56,7 @@ type parsed struct { } func newWorld() *world { - return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}} + return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}} } // put takes a change of a watched bucket in: a value, or nil for a key deleted or purged. @@ -71,6 +71,7 @@ func (w *world) put(bucket, key string, value []byte) { return } w.values[bucket][key] = value + w.changes[bucket+"|"+key]++ if bucket == delivery.HaveBucket { if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK { list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP }) @@ -349,30 +350,55 @@ func (w *world) built(module, fp string) bool { return err == nil && b.State == delivery.BuildDone } -// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record -// (mesh-controller_builds `.build.`) once and kept; empty when the record names none. -func (w *world) sentFP(ctx context.Context, get getter, module, build string) string { - if build == "" { - return "" - } - w.mu.Lock() - fp, ok := w.sent[build] - w.mu.Unlock() - if ok { - return fp - } - value, err := get(ctx, delivery.BuildKey(module, build)) +// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it +// (mesh-controller_composed `..`), and whether it exists and can be read. +func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) { + key, err := delivery.ComposedKey(node, module, fp) if err != nil { - return "" - } - var rec delivery.BuildRecord - if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) { - return "" + return delivery.ComposedDigest{}, false } w.mu.Lock() - w.sent[build] = rec.SourceFingerprint + value, ok := w.values[delivery.ControllerComposedBucket][key] w.mu.Unlock() - return rec.SourceFingerprint + if !ok { + return delivery.ComposedDigest{}, false + } + d, err := delivery.DecodeComposedDigest(value) + return d, err == nil +} + +// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `.node`). +func (w *world) nodeComposed() map[string]delivery.ComposedDigest { + out := map[string]delivery.ComposedDigest{} + for key, value := range w.read(delivery.ControllerComposedBucket) { + node, _, _, isNode, err := delivery.ParseComposedKey(key) + if err != nil || !isNode { + continue + } + if d, err := delivery.DecodeComposedDigest(value); err == nil { + out[node] = d + } + } + return out +} + +// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one. +func (w *world) answer(key string) (delivery.Answer, bool) { + w.mu.Lock() + value, ok := w.values[delivery.AnswersBucket][key] + w.mu.Unlock() + if !ok { + return delivery.Answer{}, false + } + a, err := delivery.DecodeAnswer(value) + return a, err == nil +} + +// changed is how many values the key has taken in this process (see world.changes). +func (w *world) changed(bucket, key string) uint64 { + w.mu.Lock() + defer w.mu.Unlock() + return w.changes[bucket+"|"+key] } // runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go index 5211c7a..ecea239 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -111,3 +111,17 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) { t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"]) } } + +// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen. +func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) { + w := newWorld() + w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`)) + w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`)) + w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`)) + if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 { + t.Errorf("changed = %d, want 2", got) + } + if got := w.changed(delivery.HaveBucket, "gone"); got != 0 { + t.Errorf("changed = %d for a key never seen", got) + } +} diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 6d197f7..781795a 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-assigner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 0688b07..d36d16c 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 h1:ZEEeYibEPD1rngOAqbODzRY5rUStGC46OpyQwtNCYbs= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/asks.go b/modules/mesh-delivery-assigner/internal/assign/asks.go index 52d7cb7..a3ca7ff 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks.go @@ -2,6 +2,8 @@ package assign import ( "fmt" + "regexp" + "strings" "time" "git.novox.be/novox/mesh-sdk/go/asks" @@ -9,17 +11,16 @@ import ( ) // HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep -// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer -// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6. +// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk { return delivery.AssignerAsk{Ask: asks.Ask{ ID: delivery.AskingID(key, 0), Headline: fmt.Sprintf("%s is held at %s", module, short(fp)), Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+ - "no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)), + "no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)), Options: []asks.Option{ {ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge}, - {ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge}, + {ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge}, }, Who: asks.Operator, Expires: now.Add(7 * 24 * time.Hour), @@ -27,3 +28,147 @@ func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk { About: key, }} } + +// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node. +const ( + OptionRestore = "restore" + OptionLeaveStopped = "leave-stopped" +) + +// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is +// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the +// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is. +func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk { + return delivery.AssignerAsk{Ask: asks.Ask{ + ID: delivery.AskingID(key, 0), + Headline: fmt.Sprintf("%s failed its health check on %s", module, node), + Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+ + "would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words), + Options: []asks.Option{ + {ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge}, + {ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge}, + }, + Who: asks.Operator, + Expires: now.Add(7 * 24 * time.Hour), + OnExpiry: "the module stays as it is on the node", + About: key, + }} +} + +// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or +// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's +// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding +// data, whose restore ask waits for the operator. +func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition { + key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2) + subject := asks.Scope{Scope: "module", ID: module, Machine: node} + if !raised { + return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared} + } + c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised, + Summary: fmt.Sprintf("%s is held at %s", module, short(fp)), + Headline: fmt.Sprintf("%s is held at %s", module, short(fp))} + what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words) + if needsYou { + c.Needs = "answer whether to restore " + module + " on " + node + c.Explanation = asks.NeedsYou + c.Needs + ". " + what + } else { + c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy." + } + return c +} + +// The options of the record ask (ADR 0539 §14) and of the backup ask (§16). +const ( + OptionYes = "yes" + OptionNo = "no" + OptionBackupAgain = delivery.BackupAgain + OptionBackupSkip = delivery.BackupSkip +) + +// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the +// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only +// after the answer yes. The way the move was judged is in the ask. +func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk { + what := map[delivery.Judgement]string{ + delivery.JudgedRecreates: "recreates a container", + delivery.JudgedMovesData: "moves the module's data", + delivery.JudgedCannot: "cannot be judged: " + need.Words, + }[need.Judgement] + return delivery.AssignerAsk{Ask: asks.Ask{ + ID: delivery.AskingID(key, 0), + Headline: fmt.Sprintf("%s waits for your yes on %s", module, node), + Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+ + "The node is told to apply it only after you say yes.", module, node, short(fp), what), + Options: []asks.Option{ + {ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge}, + {ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge}, + }, + Who: asks.Operator, + Expires: now.Add(7 * 24 * time.Hour), + OnExpiry: "the node keeps what it runs", + About: key, + }} +} + +// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1 +// give one id (delivery.AskingID), so the first asking is 0 and the next 2. +func NextBackupAsking(after int) int { + if after < 0 { + return 0 + } + return max(after+1, 2) +} + +var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`) + +// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names +// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again +// under a new number (ADR 0539 §16). +func BackupAskDigest(a delivery.AssignerAsk) string { + if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil { + return m[1] + } + return "" +} + +// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want +// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries. +func BackupAnswerWords(option string, asking int) string { + return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1) +} + +// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on +// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the +// backup's own words, from the node's have. +func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk { + return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{ + ID: delivery.AskingID(key, asking), + Headline: fmt.Sprintf("The backup of %s failed on %s", module, node), + Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)", + module, short(fp), node, words, module, digest), + Options: []asks.Option{ + {ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge}, + {ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge}, + }, + Who: asks.Operator, + Expires: now.Add(7 * 24 * time.Hour), + OnExpiry: "the module keeps running what it has", + About: key, + }} +} + +// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16), +// naming the node, the module and the backup's words; cleared under the same key. +func BackupCondition(node, module, fp, words string, raised bool) asks.Condition { + key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2) + subject := asks.Scope{Scope: "module", ID: module, Machine: node} + if !raised { + return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared} + } + headline := fmt.Sprintf("The backup of %s failed on %s", module, node) + needs := "answer whether to back up again or go on without a backup" + return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised, + Summary: headline, Headline: headline, Needs: needs, + Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."} +} diff --git a/modules/mesh-delivery-assigner/internal/assign/asks_test.go b/modules/mesh-delivery-assigner/internal/assign/asks_test.go index 7de1348..f734ad2 100644 --- a/modules/mesh-delivery-assigner/internal/assign/asks_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/asks_test.go @@ -1,9 +1,11 @@ package assign import ( + "strings" "testing" "time" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -26,3 +28,99 @@ func TestTheHoldAskIsValid(t *testing.T) { t.Errorf("the view may not answer it: %v", err) } } + +// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at +// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped. +func TestTheRestoreAskIsValid(t *testing.T) { + key, err := delivery.RestoreAskKey("ace", "mail", fNew) + if err != nil { + t.Fatal(err) + } + now := time.Now() + a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now) + if err := a.Ask.Check(now); err != nil { + t.Errorf("the router refuses the ask: %v", err) + } + if _, err := delivery.EncodeAssignerAsk(key, a); err != nil { + t.Errorf("the ask cannot be written: %v", err) + } + if _, err := delivery.ViewAnswerKey(key); err != nil { + t.Errorf("the view may not answer it: %v", err) + } + var ids []string + for _, o := range a.Ask.Options { + ids = append(ids, o.ID) + } + if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped { + t.Errorf("options = %v", ids) + } + if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") { + t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation) + } +} + +// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own +// name (ADR 0437 clause 9), and is cleared under the same key. +func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) { + c := HeldCondition("mail", fNew, "ace", "connection refused", true, true) + if _, err := asks.EncodeCondition(c); err != nil { + t.Fatalf("the condition is refused: %v", err) + } + if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") { + t.Errorf("condition = %+v", c) + } + if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key { + t.Errorf("cleared = %+v", got) + } +} + +// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in +// the view, and name the move and the node. +func TestTheRecordAndBackupAsksAreValid(t *testing.T) { + now := time.Now() + rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64)) + if err != nil { + t.Fatal(err) + } + r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now) + bkey, err := delivery.BackupAskKey("ace", "mail", fNew) + if err != nil { + t.Fatal(err) + } + dgst := "sha256:" + strings.Repeat("b", 64) + b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now) + for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} { + if err := a.Ask.Check(now); err != nil { + t.Errorf("%s: the router refuses the ask: %v", key, err) + } + if _, err := delivery.EncodeAssignerAsk(key, a); err != nil { + t.Errorf("%s: cannot be written: %v", key, err) + } + if _, err := delivery.ViewAnswerKey(key); err != nil { + t.Errorf("%s: the view may not answer it: %v", key, err) + } + } + if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes { + t.Errorf("the record ask = %+v", r.Ask) + } + if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip { + t.Errorf("the backup ask = %+v", b.Ask) + } + // the id binds the ask to the composition and to its number (review round 1, items 3 and 4) + if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID { + t.Errorf("asked again under the same id") + } + if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 { + t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2)) + } + if BackupAskDigest(b) != dgst { + t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst) + } + if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) { + t.Errorf("two answers of the same option are one want value") + } + c := BackupCondition("ace", "mail", fNew, "no space left", true) + if _, err := asks.EncodeCondition(c); err != nil { + t.Errorf("the backup condition is refused: %v", err) + } +} diff --git a/modules/mesh-delivery-assigner/internal/assign/compare.go b/modules/mesh-delivery-assigner/internal/assign/compare.go deleted file mode 100644 index 952acd3..0000000 --- a/modules/mesh-delivery-assigner/internal/assign/compare.go +++ /dev/null @@ -1,52 +0,0 @@ -package assign - -import ( - "fmt" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its -// assignment names, and that build's fingerprint of the new form (empty when its record names none). -type Sent struct { - Sends bool - Build string - FP string -} - -// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4), -// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and -// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the -// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said -// in words, for acceptance test 17 of design 47 to read. -func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) { - var d delivery.Difference - removed := want != nil && want.Removed - if want != nil && !removed { - d.Want = want.FP - } - d.WantRemoved = removed - if !sent.Sends { - if want == nil || removed { - return delivery.Difference{}, false - } - d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing" - return d, true - } - d.Sent, d.SentBuild = sent.FP, sent.Build - switch { - case sent.FP == "": - d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build) - case removed: - d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP)) - case sent.FP == d.Want: - return delivery.Difference{}, false - case hold.Rule != 0 && sent.FP == newest: - d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words) - case d.Want == "": - d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP)) - default: - d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want)) - } - return d, true -} diff --git a/modules/mesh-delivery-assigner/internal/assign/compare_test.go b/modules/mesh-delivery-assigner/internal/assign/compare_test.go deleted file mode 100644 index da4a14f..0000000 --- a/modules/mesh-delivery-assigner/internal/assign/compare_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package assign - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3. -func TestADifferenceExplainedByTheCanary(t *testing.T) { - m, n := ready() - n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} - out := Wants(m, n, wantedOld(), nil) - want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary} - d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"]) - if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew { - t.Fatalf("difference = %+v, %v; want rule 3", d, ok) - } - if _, err := delivery.EncodeDifference(d); err != nil { - t.Errorf("the difference cannot be written: %v", err) - } -} - -// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written -// unexplained, with words, and is writable. -func TestCompare(t *testing.T) { - w := delivery.Want{FP: fNew} - if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok { - t.Errorf("an equal want and send is a difference") - } - if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok { - t.Errorf("no want and nothing sent is a difference") - } - removed := delivery.Want{Removed: true} - if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok { - t.Errorf("a removed want and nothing sent is a difference") - } - old := delivery.Want{FP: fOld} - for name, c := range map[string]struct { - want *delivery.Want - sent Sent - }{ - "sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}}, - "names no fingerprint": {&w, Sent{Sends: true, Build: "b"}}, - "sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}}, - "sends nothing": {&w, Sent{}}, - "no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}}, - "no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}}, - } { - d, ok := Compare(c.want, c.sent, fNew, Holding{}) - if !ok || !d.Unexplained || d.Words == "" { - t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok) - continue - } - if _, err := delivery.EncodeDifference(d); err != nil { - t.Errorf("%s: the difference cannot be written: %v", name, err) - } - } -} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 524f823..2f774a3 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -20,6 +20,24 @@ type Module struct { Granted bool Manifest catalogue.Manifest Nodes []string + // Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its + // digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6). + Composed func(node, fp string) (delivery.ComposedDigest, bool) + // Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition + // with this digest (ADR 0539 §14). Nil: none did. + Allowed func(node, digest string) bool + // Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed. + Released bool + // Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back + // as any module is, losing the writes made since (ADR 0539 §19). Nil: none did. + Restored func(node string) bool +} + +func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) { + if m.Composed == nil { + return delivery.ComposedDigest{}, false + } + return m.Composed(node, fp) } // Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have. @@ -72,6 +90,9 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) { if !m.Granted { return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP)) } + if _, ok := m.composition(node, m.FP); !ok { + return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP)) + } return true, 0, "" } @@ -117,10 +138,24 @@ type Outcome struct { Wants map[string]delivery.Want // Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet. Held *delivery.Held - // Rules is, by node, the rule holding the node back from m.FP and why, for the comparison. + // Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14). + Record []RecordNeed + // Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set + // back: the operator is asked whether to restore each (ADR 0539 §19). + Restore []string + // Rules is, by node, the rule holding the node back from m.FP and why. Rules map[string]Holding } +// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the +// controller's judgement of the move and its words. +type RecordNeed struct { + Node string + Digest string + Judgement delivery.Judgement + Words string +} + // Holding is a rule holding a node back, and why. type Holding struct { Rule int @@ -130,12 +165,12 @@ type Holding struct { // Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written // before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed // health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a -// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A +// module holding data is not set back but asked about (Restore). A // node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4). func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome { out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}} held := m.Held - if !held && m.FP != "" { + if !held && !m.Released && m.FP != "" { for _, node := range slices.Sorted(slices.Values(m.Nodes)) { if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed { out.Held = &delivery.Held{Node: node, Words: h.Detail} @@ -152,30 +187,57 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st } before, had := wants[node] var next delivery.Want + // A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the + // node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the + // bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands. + if w, ok := m.userListRevision(node, before, had, n); ok { + if !had || !delivery.SameWant(normal(before), normal(w)) { + out.Wants[node] = w + } + continue + } switch ok, rule, words := Holds(mm, node, n); { case ok: - next = delivery.Want{FP: m.FP} + d, _ := m.composition(node, m.FP) + next = delivery.Want{FP: m.FP, Composed: d.Digest} + if had && !before.Removed && before.FP == m.FP && before.Composed == d.Digest { + next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16) + } + running, ran := n.Haves[node][m.Name] + if need, wait := m.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait { + if need != nil { + out.Record = append(out.Record, *need) + } + continue + } case held && had && before.FP == m.FP: out.Rules[node] = Holding{Rule: rule, Words: words} last := lastHealthy[node] switch { - case keepsData(m.Manifest): + case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)): next = before - next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP)) + next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP)) + out.Restore = append(out.Restore, node) case last == "": next = before next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node) default: - next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words} + d, there := m.composition(node, last) + if !there { + next = before + next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node) + break + } + next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words} } default: out.Rules[node] = Holding{Rule: rule, Words: words} if !had || before.Removed { continue } - next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words} + next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words} if before.FP == m.FP { - next = delivery.Want{FP: m.FP} + next = delivery.Want{FP: m.FP, Composed: before.Composed} } } if !had || !delivery.SameWant(normal(before), normal(next)) { @@ -203,3 +265,67 @@ func short(fp string) string { } return part[:12] } + +// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus, +// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of +// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the +// same fingerprint judged neither (a user list only, §7). +func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) { + if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) { + return nil, false + } + // Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition + // (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition + // unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that + // differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits. + if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged { + return nil, false + } + // The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one). + prev := before.FP + if !had { + prev = running + } + if catalogue.ProvidesBus(m.Manifest) && prev != m.FP { + return nil, true // the bus is never moved by the assigner; the operator writes its want (§15) + } + if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged { + return nil, false + } + if m.Allowed != nil && m.Allowed(node, d.Digest) { + return nil, false + } + return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true +} + +// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the +// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it. +func (m Module) recorded(d delivery.ComposedDigest) bool { + if d.Policy != "" { + return d.Policy == delivery.PolicyRecord + } + policy, _, _ := catalogue.DerivedUpgrade(m.Manifest) + return policy == catalogue.PolicyRecord +} + +// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition +// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node +// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none. +func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) { + if !catalogue.ProvidesBus(m.Manifest) { + return delivery.Want{}, false + } + running, ran := n.Haves[node][m.Name] + if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) { + return delivery.Want{}, false + } + d, ok := m.composition(node, running.FP) + if !ok || !d.UserListOnly { + return delivery.Want{}, false + } + w := delivery.Want{FP: running.FP, Composed: d.Digest} + if had && before.Composed == d.Digest { + w.Backup, w.Words = before.Backup, before.Words + } + return w, true +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index c0ff7df..31ff233 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -1,6 +1,7 @@ package assign import ( + "slices" "strings" "testing" @@ -18,7 +19,7 @@ var ( // ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider // store healthy on ace and the canary ace running fNew healthy. func ready() (Module, Nodes) { - m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Nodes: []string{"g14", "ace"}, + m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"}, Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}} n := Nodes{ Online: map[string]bool{"ace": true, "g14": true}, @@ -31,8 +32,16 @@ func ready() (Module, Nodes) { return m, n } +// dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated. +func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) } + +// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither. +func composedEverywhere(_, fp string) (delivery.ComposedDigest, bool) { + return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, true +} + func wantedOld() map[string]delivery.Want { - return map[string]delivery.Want{"ace": {FP: fOld}, "g14": {FP: fOld}} + return map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}, "g14": {FP: fOld, Composed: dig(fOld)}} } // 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so @@ -41,7 +50,7 @@ func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) { m, n := ready() out := Wants(m, n, wantedOld(), nil) for _, node := range []string{"ace", "g14"} { - if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 { + if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) { t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew) } } @@ -59,6 +68,9 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits} }, delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false }, + delivery.RuleComposed: func(m *Module, _ *Nodes) { + m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false } + }, } { m, n := ready() breakIt(&m, &n) @@ -98,11 +110,11 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) { func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { m, n := ready() n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"} - out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}, "g14": {FP: fOld}}, map[string]string{"ace": fOld}) + out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld}) if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" { t.Fatalf("held = %+v", out.Held) } - if w := out.Wants["ace"]; w.FP != fOld || w.Newest != fNew || w.Rule != delivery.RuleNotHeld { + if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld { t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w) } if w, ok := out.Wants["g14"]; ok && w.FP == fNew { @@ -122,6 +134,20 @@ func TestADataModuleIsNotSetBack(t *testing.T) { if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") { t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w) } + // 1906/D3: and the node is to be asked whether to restore it + if !slices.Equal(out.Restore, []string{"ace"}) { + t.Errorf("restore asks = %v, want ace", out.Restore) + } +} + +// 1906/D3: a module without data is set back and asked about nothing. +func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) { + m, n := ready() + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed} + out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld}) + if len(out.Restore) != 0 { + t.Errorf("restore asks = %v", out.Restore) + } } // 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that @@ -129,7 +155,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) { func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) { m, n := ready() m.Nodes = []string{"ace"} - out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew}, "g14": {FP: fOld}}, nil) + out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, nil) if w := out.Wants["g14"]; !w.Removed { t.Errorf("g14's want = %+v, want removed", w) } @@ -137,3 +163,360 @@ func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) { t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"]) } } + +// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its +// digest once it exists; a node with no want before still gets none until then. +func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) { + m, n := ready() + have := map[string]bool{} + m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) { + if !have[node+"|"+fp] { + return delivery.ComposedDigest{}, false + } + return composedEverywhere(node, fp) + } + out := Wants(m, n, wantedOld(), nil) + for node, w := range out.Wants { + if w.FP == fNew { + t.Errorf("%s's want moved to F before its composition exists: %+v", node, w) + } + } + if r := out.Rules["ace"]; r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "ace") { + t.Errorf("ace is held back by %+v, want rule 7 naming the node", r) + } + // and a node that never had a want gets none + if out := Wants(m, n, nil, nil); len(out.Wants) != 0 { + t.Errorf("wants without a composition: %+v", out.Wants) + } + have["ace|"+fNew] = true + out = Wants(m, n, wantedOld(), nil) + if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) { + t.Errorf("ace's want = %+v, want F naming the composition's digest", w) + } + if w, ok := out.Wants["g14"]; ok && w.FP == fNew { + t.Errorf("g14, whose composition does not exist, moved: %+v", w) + } +} + +// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want, +// by the same rules: written when they hold, and not while one does not. +func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) { + m, n := ready() + revised := "sha256:" + strings.Repeat("9", 64) + m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) { + return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedNeither}, true + } + running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}} + out := Wants(m, n, running, nil) + for _, node := range []string{"ace", "g14"} { + if w := out.Wants[node]; w.FP != fNew || w.Composed != revised { + t.Errorf("%s's want = %+v, want F naming the revised digest", node, w) + } + } + m.Granted = false + out = Wants(m, n, running, nil) + if w, ok := out.Wants["ace"]; ok && w.Composed == revised { + t.Errorf("a revision was written while a rule holds: %+v", w) + } +} + +// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so. +func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) { + m, n := ready() + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"} + running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}} + m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) { + if fp == fOld { + return delivery.ComposedDigest{}, false + } + return composedEverywhere(node, fp) + } + out := Wants(m, n, running, map[string]string{"ace": fOld}) + if w := out.Wants["ace"]; w.FP == fOld { + t.Errorf("set back to a fingerprint whose composition is gone: %+v", w) + } + m.Composed = composedEverywhere + out = Wants(m, n, running, map[string]string{"ace": fOld}) + if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) { + t.Errorf("set-back want = %+v", w) + } +} + +// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F). +func movingTo(j delivery.Judgement) (Module, Nodes, map[string]delivery.Want) { + m, n := ready() + m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) { + d := delivery.ComposedDigest{Digest: dig(fp), Judgement: j} + if j != delivery.JudgedNeither { + d.Against, d.Words = dig(fOld), "judged" + } + return d, true + } + return m, n, wantedOld() +} + +func recording(m *Module) { + m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"} +} + +// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be +// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint. +func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) { + for _, j := range []delivery.Judgement{delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot} { + m, n, wanted := movingTo(j) + recording(&m) + out := Wants(m, n, wanted, nil) + if w, ok := out.Wants["ace"]; ok { + t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w) + } + var ace *RecordNeed + for i := range out.Record { + if out.Record[i].Node == "ace" { + ace = &out.Record[i] + } + } + if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j { + t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record) + } + // a new revision at the same fingerprint is asked too + same := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}} + if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) { + t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out) + } + m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) } + out = Wants(m, n, wanted, nil) + if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) { + t.Errorf("%s: after the yes ace's want = %+v", j, w) + } + } +} + +// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module +// that rolls (not record) is never asked. +func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) { + m, n, wanted := movingTo(delivery.JudgedNeither) + recording(&m) + if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 { + t.Errorf("neither: %+v", out) + } + m, n, _ = movingTo(delivery.JudgedRecreates) + recording(&m) + delete(n.Haves["ace"], "mail") + if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 { + t.Errorf("a node that never ran it: %+v", out) + } + // but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below) + m, n, _ = movingTo(delivery.JudgedCannot) + recording(&m) + n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew || len(out.Record) == 0 { + t.Errorf("a node that runs it with no want: %+v", out) + } + m, n, wanted = movingTo(delivery.JudgedRecreates) + if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 { + t.Errorf("a rolled module: %+v", out) + } +} + +// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its +// composition at the same fingerprint that only changes the user list (judged neither) is written at once. +func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) { + m, n, _ := movingTo(delivery.JudgedNeither) + m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} + if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew { + t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"]) + } + running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}} + out := Wants(m, n, running, nil) + if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) { + t.Errorf("a user-list revision of the bus: ace's want = %+v", w) + } + if len(out.Record) != 0 { + t.Errorf("a user-list revision was asked: %+v", out.Record) + } +} + +// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays. +func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) { + m, n := ready() + running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew), Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}} + if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok { + t.Errorf("the answer was erased: %+v", w) + } +} + +// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the +// wants move by the rules again. +func TestAReleasedHoldIsNotHeldAgain(t *testing.T) { + m, n := ready() + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"} + m.Released = true + out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleNotHeld}}, map[string]string{"ace": fOld}) + if out.Held != nil { + t.Errorf("a released fingerprint was held again: %+v", out.Held) + } + if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) || w.Rule != 0 { + t.Errorf("ace's want = %+v, want the held fingerprint written again", w) + } +} + +// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes; +// answered "leave stopped" (or not answered) it stays. +func TestARestoreAnswerSetsADataModuleBack(t *testing.T) { + m, n := ready() + m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed} + running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}} + m.Restored = func(node string) bool { return false } + if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.FP == fOld { + t.Errorf("set back without the operator's restore: %+v", w) + } + m.Restored = func(node string) bool { return node == "ace" } + if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) { + t.Errorf("after restore ace's want = %+v", w) + } +} + +// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included; +// a composition with none (made by hand) falls back to what the manifest derives. +func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) { + m, n, wanted := movingTo(delivery.JudgedMovesData) + with := func(policy, from string) { + inner := m.Composed + m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) { + d, ok := inner(node, fp) + d.Policy, d.PolicyFrom = policy, from + return d, ok + } + } + with(delivery.PolicyRecord, "person") // a roll module the operator set to record + if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].FP == fNew { + t.Errorf("the operator's record policy was not honoured: %+v", out) + } + recording(&m) // the manifest says record, the operator chose roll + with("roll", "person") + if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew { + t.Errorf("the operator's roll policy was not honoured: %+v", out) + } + with("", "") // made by hand: the manifest decides + if out := Wants(m, n, wanted, nil); len(out.Record) == 0 { + t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out) + } +} + +// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one +// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred. +func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) { + m, n, _ := movingTo(delivery.JudgedNeither) + m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew { + t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok) + } + n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok { + t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w) + } +} + +// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written +// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it +// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against +// the composition the node runs, policy record, UserListOnly set. +func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) { + m, n, _ := movingTo(delivery.JudgedRecreates) + m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} + userList := "sha256:" + strings.Repeat("7", 64) + m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) { + d := delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted", + Policy: delivery.PolicyRecord, PolicyFrom: "the bus"} + if fp == fOld { + d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list + } + return d, true + } + n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + for name, wanted := range map[string]map[string]delivery.Want{ + "a want at the fingerprint it runs": {"ace": {FP: fOld, Composed: dig(fOld)}}, + "no readable want": nil, + } { + out := Wants(m, n, wanted, nil) + if w := out.Wants["ace"]; w.FP != fOld || w.Composed != userList { + t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w) + } + if len(out.Record) != 0 { + t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record) + } + } + // the same flag on a module that is not the bus changes nothing + m.Manifest.Provides = nil + if out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList { + t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"]) + } +} + +// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine +// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F +// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the +// declaration it kept); a new fingerprint still waits for its ask. +func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) { + for name, bus := range map[string]bool{"a record module": false, "the bus": true} { + // only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold + unchanged := func(node, fp string) (delivery.ComposedDigest, bool) { + return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedUnchanged}, true + } + m, n, _ := movingTo(delivery.JudgedCannot) + recording(&m) + m.Nodes = []string{"ace"} + if bus { + m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} + } + m.Composed = unchanged + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} // no composition named + out := Wants(m, n, nil, nil) + if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) { + t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w) + } + if len(out.Record) != 0 { + t.Errorf("%s: asked at the switch: %+v", name, out.Record) + } + // a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge, + // still wait for the ask + for what, j := range map[string]delivery.Judgement{"differing": delivery.JudgedRecreates, "unjudged": delivery.JudgedCannot, "judged neither": delivery.JudgedNeither} { + m2, n2, _ := movingTo(j) + recording(&m2) + m2.Nodes = []string{"ace"} + if bus { + m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} + } + n2.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + out := Wants(m2, n2, nil, nil) + if j == delivery.JudgedNeither { + if out.Wants["ace"].FP != fNew || len(out.Record) != 0 { + t.Errorf("%s, %s: %+v", name, what, out) + } + continue + } + if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 { + t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record) + } + } + // a have that names a composition is judged as before + m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) { + return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "judged"}, true + } + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew && len(out.Record) == 0 { + t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out) + } + // a new fingerprint still asks (the bus is barred instead) + n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + out = Wants(m, n, nil, nil) + if w, ok := out.Wants["ace"]; ok { + t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w) + } + if !bus && len(out.Record) == 0 { + t.Errorf("%s: a new fingerprint was not asked", name) + } + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index db46536..cd6fe55 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest { // 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the // manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps -// its three states, and holds a credential of its own. +// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own. func TestManifest(t *testing.T) { m := load(t) if m.Module != "mesh-delivery-assigner" { @@ -32,9 +32,9 @@ func TestManifest(t *testing.T) { } reads := slices.Clone(m.Reads) slices.Sort(reads) - want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants", + want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"} + "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"} if !reflect.DeepEqual(reads, want) { t.Errorf("reads = %v, want exactly %v", reads, want) } @@ -51,9 +51,10 @@ func TestManifest(t *testing.T) { if m.SecretsOwner != "mesh-delivery-assigner" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // 1622: the assigner puts its compose asks on the Go build seat, as the checker puts its checks. - if !slices.Equal(m.Uses, []string{"node-build-go"}) { - t.Errorf("uses = %v, want exactly node-build-go", m.Uses) + // 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the + // checker puts its checks. + if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) { + t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses) } if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 { t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes) diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index b07feba..9de9024 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -6,16 +6,19 @@ "mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", + "mesh-controller.composed", "mesh-controller.grants", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have" ], "uses": [ - "node-build-go" + "node-build-go", + "operator-channel" ], "state": [ { "name": "asks", "history": 5 }, diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go index 448979d..d81d219 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go @@ -121,5 +121,12 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue. if err != nil { return catalogue.Manifest{}, err } - return catalogue.ParseManifest([]byte(kept.JSON)) + return parseManifest(kept.JSON) +} + +// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that +// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this +// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7). +func parseManifest(raw string) (catalogue.Manifest, error) { + return catalogue.ParseManifestForInputs([]byte(raw)) } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index cea8b84..066488f 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -6,7 +6,6 @@ import ( "strings" "sync" - "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" @@ -93,7 +92,7 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted if mk, err := delivery.ManifestsKey(m.Manifest); err == nil { if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found { if kept, err := delivery.DecodeManifest(raw); err == nil { - if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil { + if man, err := parseManifest(kept.JSON); err == nil && man.Build != nil { for _, a := range man.Build.Artifacts { if a.Language != "" { set[a.Language] = true diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index 126b348..0c04f87 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "os" "strings" "testing" @@ -112,3 +113,31 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t t.Errorf("another repository: %v", got) } } + +// A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the +// catalogue's predefined.json this process never holds: the indexer and the assigner read it with +// catalogue.ParseManifestForInputs (issues 1751, 1906). xterm's module.json from mesh-catalog at 835bf22b. +func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) { + raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json") + if err != nil { + t.Fatal(err) + } + manifest := string(raw) + data, err := delivery.EncodeManifest(delivery.Manifest{Module: "xterm", JSON: manifest}) + if err != nil { + t.Fatal(err) + } + store := memManifests{delivery.ManifestDigest([]byte(manifest)): data} + var said []string + r := &repoModules{manifests: store, logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} + putModule(t, r, "xterm", "novox/mesh-catalog", manifest) + r.BuildModules("novox/mesh-catalog") + for _, s := range said { + if strings.Contains(s, "cannot be read") { + t.Errorf("the manifest was not read: %s", s) + } + } + if _, err := parseManifest(manifest); err != nil { + t.Errorf("manifestReader: %v", err) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/testdata/catalogue-835bf22b-xterm.module.json b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/testdata/catalogue-835bf22b-xterm.module.json new file mode 100644 index 0000000..b65be8e --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/testdata/catalogue-835bf22b-xterm.module.json @@ -0,0 +1,236 @@ +{ + "module": "xterm", + "version": "1", + "moved": true, + "capabilities": [ + "package-manager" + ], + "requires": [ + "x11-display" + ], + "claims": [ + { + "name": "node-terminal-emulator", + "scope": "node", + "serves": [ + "open" + ] + } + ], + "tools": [ + "xterm_font", + "xterm_colours" + ], + "config": { + "face": { + "type": "string", + "description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).", + "pattern": "[A-Za-z0-9 ._-]{1,64}", + "default": "JetBrainsMono Nerd Font" + }, + "size": { + "type": "number", + "description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.", + "minimum": 6, + "maximum": 72, + "predefined": "font-size", + "default": 12 + }, + "save-lines": { + "type": "number", + "description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.", + "minimum": 0, + "maximum": 1000000, + "default": 10000 + }, + "term-name": { + "type": "string", + "description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.", + "pattern": "[A-Za-z0-9._-]{1,64}", + "default": "xterm-256color" + }, + "render-font": { + "type": "boolean", + "description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.", + "default": true + }, + "bold-mode": { + "type": "boolean", + "description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.", + "default": false + }, + "always-bold-mode": { + "type": "boolean", + "description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.", + "default": false + }, + "allow-bold-fonts": { + "type": "boolean", + "description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.", + "default": true + }, + "background": { + "type": "string", + "description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#000000" + }, + "foreground": { + "type": "string", + "description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#a8a8a8" + }, + "color0": { + "type": "string", + "description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#000000" + }, + "color1": { + "type": "string", + "description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#9e1828" + }, + "color2": { + "type": "string", + "description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#008800" + }, + "color3": { + "type": "string", + "description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#d2bb4b" + }, + "color4": { + "type": "string", + "description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#414171" + }, + "color5": { + "type": "string", + "description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#963c59" + }, + "color6": { + "type": "string", + "description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#418179" + }, + "color7": { + "type": "string", + "description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#bebebe" + }, + "color8": { + "type": "string", + "description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#666666" + }, + "color9": { + "type": "string", + "description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#bc5766" + }, + "color10": { + "type": "string", + "description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#61a171" + }, + "color11": { + "type": "string", + "description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#e7db52" + }, + "color12": { + "type": "string", + "description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#5085af" + }, + "color13": { + "type": "string", + "description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#a97a99" + }, + "color14": { + "type": "string", + "description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#6ba4a4" + }, + "color15": { + "type": "string", + "description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.", + "pattern": "#[0-9a-fA-F]{6}", + "default": "#ffffff" + }, + "copy-binding": { + "type": "string", + "description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.", + "pattern": "[^:\\\\\\n]+", + "default": "Ctrl Shift ~Alt C" + }, + "paste-binding": { + "type": "string", + "description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.", + "pattern": "[^:\\\\\\n]+", + "default": "Ctrl Shift ~Alt V" + } + }, + "items": [ + { + "collection": "environment", + "name": "terminal", + "attributes": { + "kind": "variable", + "name": "TERMINAL" + }, + "template": "files/env-terminal" + }, + { + "collection": "shell:xresources:normal", + "name": "xresources-normal", + "template": "files/xresources-normal" + } + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "xterm" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/xterm-tools", + "binary": "xterm-tools", + "loads": [ + "xterm-tools" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 645c726..c277b4b 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "os" + "strings" stdio "git.novox.be/novox/mesh-sdk/go" "mesh-delivery-view/internal/view" @@ -82,9 +83,50 @@ func Tools(r view.Reader) []stdio.Tool { commit, _ := args["commit"].(string) return r.BuilderBuild(module, fp2, commit) }}, + {Name: view.WantsReadTool, + Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want..) and " + + "held fingerprints by key (held..). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.", + Input: map[string]any{"type": "object", "properties": nodeModule}, + Run: func(args map[string]any) (any, error) { + node, module := nodeModuleOf(args) + return r.Wants(node, module) + }}, + {Name: view.WantReadTool, + Description: "One want of the assigner as stored: what the node is to run of the module (fingerprint, the newest it is held back from and the rule, removed, words). Read-only.", + Input: map[string]any{"type": "object", "required": []string{"node", "module"}, "properties": nodeModule}, + Run: func(args map[string]any) (any, error) { + node, module := nodeModuleOf(args) + return r.Want(node, module) + }}, + {Name: view.DifferencesReadTool, + Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " + + "by key (.), with the rule that explains each or unexplained. Narrow by node and module. Read-only.", + Input: map[string]any{"type": "object", "properties": nodeModule}, + Run: func(args map[string]any) (any, error) { + node, module := nodeModuleOf(args) + return r.Differences(node, module) + }}, + {Name: view.HavesReadTool, + Description: "What each node has, as the node-engine stored it in node-engine_have: by key . (the fingerprint it runs, its phase and health), " + + "and .node (the node's own facts). Narrow by node and module. Read-only.", + Input: map[string]any{"type": "object", "properties": nodeModule}, + Run: func(args map[string]any) (any, error) { + node, module := nodeModuleOf(args) + return r.Haves(node, module) + }}, } } +var nodeModule = map[string]any{ + "node": map[string]any{"type": "string", "description": "only this node's keys"}, + "module": map[string]any{"type": "string", "description": "only this module's keys"}} + +func nodeModuleOf(args map[string]any) (string, string) { + node, _ := args["node"].(string) + module, _ := args["module"].(string) + return strings.TrimSpace(node), strings.TrimSpace(module) +} + func read(state stdio.KeptState) view.StateRead { return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) { e, err := state.Get(key) @@ -102,7 +144,8 @@ func read(state stdio.KeptState) view.StateRead { } func main() { - r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))} + r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")), + WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))} if err := stdio.Serve("", Tools(r)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index 0c13bdf..008e7d1 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -264,3 +264,68 @@ func TestCandidateToolsAnswerTheStoredValue(t *testing.T) { t.Errorf("by module = %v", c) } } + +// 1906/D1 AC1: assigner_wants, assigner_want, assigner_differences and node_haves answer as stored, narrowed by node and module. +func keptReader() view.Reader { + st := func(f fake) view.StateRead { return f.reader().State } + return view.Reader{ + WantsState: st(fake{ + "want.ace.zsh": `{"fp":"` + fp2A + `"}`, + "want.novox.zsh": `{"fp":"` + fp2B + `"}`, + "want.ace.netcheck": `{"fp":"` + fp2A + `","rule":3}`, + "held." + keyB: `{"node":"ace","words":"failed"}`, + }), + DifferencesState: st(fake{"ace.zsh": `{"unexplained":true}`, "novox.zsh": `{"rule":3}`}), + HavesState: st(fake{"ace.zsh": `{"fp":"x"}`, "ace.node": `{"facts":1}`, "novox.zsh": `{"fp":"y"}`}), + } +} + +func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) { + r := keptReader() + out, err := run(t, r, view.WantsReadTool, map[string]any{}) + if err != nil { + t.Fatal(err) + } + a := out.(view.WantsAnswer) + if len(a.Wants) != 3 || len(a.Held) != 1 || string(a.Wants["want.ace.zsh"]) != `{"fp":"`+fp2A+`"}` { + t.Errorf("all = %+v", a) + } + out, _ = run(t, r, view.WantsReadTool, map[string]any{"node": "ace"}) + if got := out.(view.WantsAnswer).Wants; len(got) != 2 { + t.Errorf("node ace = %v", got) + } + out, _ = run(t, r, view.WantsReadTool, map[string]any{"module": "zsh"}) + if got := out.(view.WantsAnswer).Wants; len(got) != 2 { + t.Errorf("module zsh = %v", got) + } +} + +func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) { + r := keptReader() + out, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "zsh"}) + if err != nil || string(out.(json.RawMessage)) != `{"fp":"`+fp2A+`"}` { + t.Errorf("want = %v %v", out, err) + } + if _, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "gone"}); err == nil || !strings.Contains(err.Error(), "no want") { + t.Errorf("a missing want: %v", err) + } +} + +func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) { + r := keptReader() + out, err := run(t, r, view.DifferencesReadTool, map[string]any{"node": "novox"}) + if err != nil { + t.Fatal(err) + } + if got := out.(view.KeptAnswer).Keys; len(got) != 1 || string(got["novox.zsh"]) != `{"rule":3}` { + t.Errorf("differences = %v", got) + } + out, _ = run(t, r, view.HavesReadTool, map[string]any{"module": "zsh"}) + if got := out.(view.KeptAnswer).Keys; len(got) != 2 { + t.Errorf("haves of zsh = %v", got) + } + out, _ = run(t, r, view.HavesReadTool, map[string]any{"node": "ace"}) + if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil { + t.Errorf("haves of ace (the node's own key too) = %v", got) + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 336b33a..69d4247 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -433,17 +433,17 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerDifferencesBucket, History: delivery.AssignerDifferencesHistory}) + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerWantsBucket, History: delivery.AssignerWantsHistory}) if err != nil { t.Fatal(err) } const fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111" const fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222" - for key, d := range map[string]delivery.Difference{ - "ace.plex": {Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}, - "g14.plex": {Want: fpOld, Sent: fpNew, Unexplained: true}, + for key, d := range map[string]delivery.Want{ + "want.ace.plex": {FP: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)}, + "want.g14.plex": {FP: fpOld, Composed: "sha256:" + strings.Repeat("c", 64), Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"}, } { - v, err := delivery.EncodeDifference(d) + v, err := delivery.EncodeWant(d) if err != nil { t.Fatal(err) } @@ -465,13 +465,13 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC t.Fatalf("%s; last: %+v", what, last) } wait(func(s view.Snapshot) bool { - d := s.Differences - return len(d) == 2 && d[0].Node == "ace" && d[0].Rule == delivery.RuleCanary && d[1].Node == "g14" && d[1].Unexplained - }, "the two differences never showed with their rules") - if err := kv.Delete(ctx, "g14.plex"); err != nil { + d := s.Wants + return len(d) == 2 && d[0].Node == "ace" && d[0].State == view.WantPending && d[1].Node == "g14" && d[1].State == view.WantHeldBack && d[1].Rule == delivery.RuleCanary + }, "the two wants never showed with where they stand") + if err := kv.Delete(ctx, "want.g14.plex"); err != nil { t.Fatal(err) } - wait(func(s view.Snapshot) bool { return len(s.Differences) == 1 }, "the difference taken away never left the list") + wait(func(s view.Snapshot) bool { return len(s.Wants) == 1 }, "the want taken away never left the list") } // TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof starts the program whole, with the assigner's asks, the view's @@ -696,3 +696,230 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC } t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog) } + +// 1906/D9, whole on the bus: when the wants and the haves make the source revision delivered, the view says it once, as its event +// `delivered` with the revision, the group and the heads, which the issue tracker derives "fix delivered" from. +func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { + srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(srv.Shutdown) + nc, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}}) + if err != nil { + t.Fatal(err) + } + kvs := map[string]jetstream.KeyValue{} + for _, name := range []string{delivery.SourceBucket, delivery.ModulesBucket, delivery.AssignerWantsBucket, delivery.HaveBucket} { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5}) + if err != nil { + t.Fatal(err) + } + kvs[name] = kv + } + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + broker := filepath.Join(dir, "broker") + cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"}) + if err := os.WriteFile(broker, cred, 0o600); err != nil { + t.Fatal(err) + } + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"} + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() + t.Cleanup(func() { stop(); <-done }) + + fp := "src2:" + strings.Repeat("2", 64) + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) + failed, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}) + good, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) + for bucket, kv := range map[string][]byte{delivery.ModulesBucket: mod} { + if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil { + t.Fatal(err) + } + } + if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil { + t.Fatal(err) + } + if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", failed); err != nil { + t.Fatal(err) + } + rev, err := kvs[delivery.SourceBucket].Put(ctx, delivery.SourceMainKey, src) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if info, err := events.Info(ctx); err != nil || info.State.Msgs != 0 { + t.Fatalf("an event was said before the revision was delivered: %v %+v", err, info.State) + } + if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", good); err != nil { + t.Fatal(err) + } + var msg *jetstream.RawStreamMsg + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && msg == nil; time.Sleep(100 * time.Millisecond) { + msg, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+DeliveredEventType) + } + if msg == nil { + t.Fatal("the delivery of the source revision was never said") + } + var e view.DeliveredEvent + if err := json.Unmarshal(msg.Data, &e); err != nil || e.Revision != rev || e.Group != "feat/x" || e.Heads["novox/web"] != strings.Repeat("1", 40) { + t.Errorf("the event = %+v, %v; want revision %d, group feat/x and the heads", e, err, rev) + } + if msg.Header.Get("x-source") != "mesh-delivery-view" || msg.Header.Get("x-event-id") == "" { + t.Errorf("the headers = %v", msg.Header) + } +} + +// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and +// healthy `settled`, a have held or failed `stuck`, each once with the body the messenger reads and an event id per type, node, module +// and composition. +func TestProcessWholeSaysModulesInMotion(t *testing.T) { + srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(srv.Shutdown) + nc, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}}) + if err != nil { + t.Fatal(err) + } + kvs := map[string]jetstream.KeyValue{} + for _, name := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} { + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5}) + if err != nil { + t.Fatal(err) + } + kvs[name] = kv + } + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + broker := filepath.Join(dir, "broker") + cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"}) + if err := os.WriteFile(broker, cred, 0o600); err != nil { + t.Fatal(err) + } + env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"} + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() + t.Cleanup(func() { stop(); <-done }) + time.Sleep(2 * time.Second) // the view has read what stood: nothing + + fpOld, fpNew := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64) + digest := "sha256:" + strings.Repeat("c", 64) + put := func(bucket, key string, v []byte, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + if _, err := kvs[bucket].Put(ctx, key, v); err != nil { + t.Fatal(err) + } + } + want := func(node, module string) { + v, err := delivery.EncodeWant(delivery.Want{FP: fpNew, Composed: digest}) + put(delivery.AssignerWantsBucket, "want."+node+"."+module, v, err) + } + have := func(node, module, fp string, phase delivery.Phase, health delivery.Health, detail string) { + v, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: phase, Health: health, Detail: detail}) + put(delivery.HaveBucket, node+"."+module, v, err) + } + have("ace", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "") + want("ace", "web") + have("g14", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "") + want("g14", "web") + + said := func() map[string]jetstream.RawStreamMsg { + out := map[string]jetstream.RawStreamMsg{} + for seq := uint64(1); ; seq++ { + m, err := events.GetMsg(ctx, seq) + if err != nil { + return out + } + out[m.Header.Get("x-event-id")] = *m + } + } + waitFor := func(id string) jetstream.RawStreamMsg { + t.Helper() + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if m, ok := said()[id]; ok { + return m + } + } + t.Fatalf("never said %s; said %v", id, func() []string { + var ids []string + for k := range said() { + ids = append(ids, k) + } + return ids + }()) + return jetstream.RawStreamMsg{} + } + hex := strings.Repeat("c", 64) + mv := waitFor("moving.ace.web." + hex) + var body view.MotionBody + if err := json.Unmarshal(mv.Data, &body); err != nil || body.Node != "ace" || body.Module != "web" || body.FromFP != fpOld || body.ToFP != fpNew || body.Composed != digest || body.BoundSeconds != view.MotionBoundSeconds || body.Since.IsZero() { + t.Errorf("moving = %+v, %v", body, err) + } + if mv.Subject != "mesh.mod.mesh-delivery-view.event.moving" { + t.Errorf("subject = %s", mv.Subject) + } + have("ace", "web", fpNew, delivery.PhaseRunning, delivery.HealthOK, "") + st := waitFor("settled.ace.web." + hex) + if err := json.Unmarshal(st.Data, &body); err != nil || body.Outcome != "running" { + t.Errorf("settled = %+v, %v", body, err) + } + have("g14", "web", fpOld, delivery.PhaseHeld, delivery.HealthWaits, "no space left") + stk := waitFor("stuck.g14.web." + hex) + if err := json.Unmarshal(stk.Data, &body); err != nil || body.Outcome != "stopped" || body.Words != "no space left" { + t.Errorf("stuck = %+v, %v", body, err) + } + time.Sleep(time.Second) + count := map[string]int{} + for id := range said() { + count[strings.SplitN(id, ".", 2)[0]]++ + } + if count["moving"] != 2 || count["settled"] != 1 || count["stuck"] != 1 { + t.Errorf("events said by type = %v, want moving 2, settled 1, stuck 1", count) + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index 8b190eb..4ae4f95 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -138,8 +138,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } model.SetWriter(kvWriter{ctx: ctx, js: js}) + model.SetMotionEvent(func(e view.MotionEvent) { + if err := emitEvent(ctx, js, cred, e.Type, e.Body, e.ID); err != nil { + logger.Printf("the %s of %s on %s could not be said: %v", e.Type, e.Body.Module, e.Body.Node, err) + } + }) + model.SetDeliveredEvent(func(e view.DeliveredEvent) { + if err := emitEvent(ctx, js, cred, DeliveredEventType, e, fmt.Sprintf("delivered.%d", e.Revision)); err != nil { + logger.Printf("the delivery of source revision %d could not be said: %v", e.Revision, err) + } + }) model.SetBacklog(view.BacklogOver(js, 5*time.Second)) - for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} { + for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} { go follow(ctx, js, bucket, model, logger) } select { @@ -204,10 +214,36 @@ func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, model return errors.New("the watch ended") } if e == nil { // the last values have all been delivered + model.Initial(bucket) continue } model.Apply(view.Entry{Bucket: bucket, Key: e.Key(), Value: e.Value(), At: e.Created(), - Deleted: e.Operation() != jetstream.KeyValuePut}) + Deleted: e.Operation() != jetstream.KeyValuePut, Revision: e.Revision()}) } } } + +// DeliveredEventType is the event the view says when a source revision is delivered: mesh.mod.mesh-delivery-view.event.delivered +// (issue 1906 D9, ADR 0539 §17). Its body is view.DeliveredEvent. +const DeliveredEventType = "delivered" + +// emitEvent publishes an event of the view's own namespace, mesh.mod..event., with the SDK's envelope headers +// (as mesh-sdk's process.Conn.Emit does) and the id as the bus's own dedup id, so the same revision said twice in the stream's +// duplicate window is kept once. +func emitEvent(ctx context.Context, js jetstream.JetStream, cred credential, eventType string, body any, id string) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + eventType) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) + return err +} diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 1711406..70dc615 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-view go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 2a3e9f3..d36d16c 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/internal/view/differences_test.go b/modules/mesh-delivery-view/internal/view/differences_test.go deleted file mode 100644 index b8fa993..0000000 --- a/modules/mesh-delivery-view/internal/view/differences_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package view - -import ( - "net/http" - "net/http/httptest" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -const ( - fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111" - fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222" -) - -// diffEntry is a key of the assigner's differences as the assigner writes it (mesh-sdk go/delivery EncodeDifference). -func differenceE(t *testing.T, node, module string, d delivery.Difference) Entry { - t.Helper() - key, err := delivery.DifferenceKey(node, module) - if err != nil { - t.Fatal(err) - } - v, err := delivery.EncodeDifference(d) - if err != nil { - t.Fatal(err) - } - return Entry{Bucket: delivery.AssignerDifferencesBucket, Key: key, Value: v, At: t0} -} - -func TestEveryDifferenceIsListedWithItsRule(t *testing.T) { // 1297/B8 AC - m := New() - fill(m, - differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}), - differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}), - differenceE(t, "ace", "old", delivery.Difference{WantRemoved: true, Sent: fpOld, Rule: delivery.RuleNotHeld}), - ) - got := m.Snapshot().Differences - if len(got) != 3 { - t.Fatalf("differences = %+v, want all three", got) - } - // ordered by node then module - if got[0].Node != "ace" || got[0].Module != "old" || got[1].Module != "plex" || got[2].Node != "g14" { - t.Errorf("order = %+v", got) - } - if r := got[1]; r.Want != fpOld || r.Sent != fpNew || r.Rule != delivery.RuleCanary || r.Unexplained || r.Words != "ace is not the canary" { - t.Errorf("row = %+v", r) - } - if !got[0].WantRemoved || got[0].Rule != delivery.RuleNotHeld { - t.Errorf("removed row = %+v", got[0]) - } - if !got[2].Unexplained || got[2].Rule != 0 { - t.Errorf("unexplained row = %+v", got[2]) - } - // a difference taken away (the want and the send agree again) leaves the list - fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "g14.plex", Deleted: true}) - if got := m.Snapshot().Differences; len(got) != 2 { - t.Errorf("after the delete = %+v", got) - } -} - -func TestAnUnreadableDifferenceIsNamedAndNotDropped(t *testing.T) { // never dropped silently - m := New() - fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "ace.plex", Value: []byte(`{"v":99}`), At: t0}) - s := m.Snapshot() - if len(s.Differences) != 0 || len(s.Unreadable) != 1 || s.Unreadable[0].Bucket != delivery.AssignerDifferencesBucket || s.Unreadable[0].Key != "ace.plex" { - t.Errorf("snapshot = %+v", s) - } -} - -func TestTableAndPageShowTheWantsAgainstTheSends(t *testing.T) { // 1297/B8 AC - m := New() - fill(m, - differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}), - differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}), - ) - var out strings.Builder - Table(&out, m.Snapshot()) - for _, w := range []string{"WANTS AGAINST SENDS", "ace", "plex", fpOld, fpNew, "rule 3", "ace is not the canary", "unexplained"} { - if !strings.Contains(out.String(), w) { - t.Errorf("table lacks %q:\n%s", w, out.String()) - } - } - for _, path := range []string{"/", "/wants"} { - rec := httptest.NewRecorder() - Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) - body := rec.Body.String() - if rec.Code != 200 { - t.Fatalf("%s = %d", path, rec.Code) - } - for _, w := range []string{"WANTS AGAINST SENDS", "rule 3", "unexplained", "ace is not the canary"} { - if !strings.Contains(body, w) { - t.Errorf("%s lacks %q:\n%s", path, w, body) - } - } - } - // With none, the page says so and does not hide the heading. - rec := httptest.NewRecorder() - Web(New()).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/wants", nil)) - if !strings.Contains(rec.Body.String(), "no difference") { - t.Errorf("empty page = %q", rec.Body.String()) - } -} diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 8f1546a..bac6085 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -33,7 +33,10 @@ type Reader struct { State StateRead // the indexer's modules Builds StateRead // the builder's builds - CandidatesState StateRead // the indexer's candidates + CandidatesState StateRead // the indexer's candidates + WantsState StateRead // the assigner's wants and holds + DifferencesState StateRead // the assigner's differences + HavesState StateRead // the node-engine's haves } // Module is the stored value of modules., exactly as the indexer wrote it. @@ -328,3 +331,124 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) { } return row, nil } + +// The read tools of the assigner's wants and differences and of the node-engine's haves (issue 1906, D1): the keys of those +// buckets answered as stored, so what the assigner wants, what it found different and what each node has can be read live. +// They write nothing. +const ( + WantsReadTool = "assigner_wants" + WantReadTool = "assigner_want" + DifferencesReadTool = "assigner_differences" + HavesReadTool = "node_haves" +) + +// WantsAnswer is the answer of assigner_wants: the wants by key (`want..`) and the holds by key +// (`held..`), each as stored, and the keys that are neither. +type WantsAnswer struct { + Wants map[string]json.RawMessage `json:"wants"` + Held map[string]json.RawMessage `json:"held"` + Unreadable []string `json:"unreadable,omitempty"` +} + +// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that +// are no `.` key. +type KeptAnswer struct { + Keys map[string]json.RawMessage `json:"keys"` + Unreadable []string `json:"unreadable,omitempty"` +} + +func (s StateRead) all(what string) (map[string]json.RawMessage, error) { + if s.Keys == nil || s.Get == nil { + return nil, fmt.Errorf("%s cannot be read here", what) + } + keys, err := s.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + out := map[string]json.RawMessage{} + for _, k := range keys { + raw, err := s.Get(k) + if err != nil { + return nil, err + } + if len(raw) > 0 { + out[k] = raw + } + } + return out, nil +} + +// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module +// matches). +func (r Reader) Wants(node, module string) (WantsAnswer, error) { + all, err := r.WantsState.all("the assigner's wants") + if err != nil { + return WantsAnswer{}, err + } + out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}} + for k, raw := range all { + if n, m, err := delivery.ParseWantKey(k); err == nil { + if (node == "" || n == node) && (module == "" || m == module) { + out.Wants[k] = raw + } + continue + } + if m, _, err := delivery.ParseHeldKey(k); err == nil { + if node == "" && (module == "" || m == module) { + out.Held[k] = raw + } + continue + } + out.Unreadable = append(out.Unreadable, k) + } + return out, nil +} + +// Want is the stored want of module on node. +func (r Reader) Want(node, module string) (json.RawMessage, error) { + key, err := delivery.WantKey(strings.TrimSpace(node), strings.TrimSpace(module)) + if err != nil { + return nil, err + } + if r.WantsState.Get == nil { + return nil, fmt.Errorf("the assigner's wants cannot be read here") + } + raw, err := r.WantsState.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("the assigner has no want %s", key) + } + return raw, nil +} + +func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) { + all, err := s.all(what) + if err != nil { + return KeptAnswer{}, err + } + out := KeptAnswer{Keys: map[string]json.RawMessage{}} + for k, raw := range all { + n, m, isNode, err := delivery.ParseHaveKey(k) + if err != nil { + out.Unreadable = append(out.Unreadable, k) + continue + } + if (node == "" || n == node) && (module == "" || (!isNode && m == module)) { + out.Keys[k] = raw + } + } + return out, nil +} + +// Differences is the assigner's differences as stored, narrowed by node and module. +func (r Reader) Differences(node, module string) (KeptAnswer, error) { + return r.kept(r.DifferencesState, "the assigner's differences", node, module) +} + +// Haves is the node-engine's haves (and each node's own key) as stored, narrowed by node and module. +func (r Reader) Haves(node, module string) (KeptAnswer, error) { + return r.kept(r.HavesState, "the haves", node, module) +} diff --git a/modules/mesh-delivery-view/internal/view/serve.go b/modules/mesh-delivery-view/internal/view/serve.go index f8f6685..f575a51 100644 --- a/modules/mesh-delivery-view/internal/view/serve.go +++ b/modules/mesh-delivery-view/internal/view/serve.go @@ -209,33 +209,42 @@ func askText(s Snapshot) [][]string { return out } -// differenceText is one line per difference: the node, the module, what the assigner wants, what the controller sends, the rule -// that explains it (or unexplained) and the assigner's words. -func differenceText(s Snapshot) [][]string { +// wantText is one line per want: the node, the module, what the assigner wants, what the node runs, where the two stand and the +// assigner's words (the rule that holds a node back, when one does). +func wantText(s Snapshot) [][]string { var out [][]string - for _, d := range s.Differences { - want, sent := d.Want, d.Sent - switch { - case d.WantRemoved: - want = "removed" - case want == "": + for _, w := range s.Wants { + want, have := w.Want, w.Have + if want == "" { want = "none" } - if sent == "" { - sent = "none" - if d.SentBuild != "" { - sent = "build " + d.SentBuild - } + if have == "" { + have = "none" } - why := "unexplained" - if !d.Unexplained { - why = fmt.Sprintf("rule %d", d.Rule) + words := w.Words + if w.Rule != 0 { + words = fmt.Sprintf("rule %d: %s", w.Rule, w.Words) } - out = append(out, []string{d.Node, d.Module, want, sent, why, d.Words}) + judged := w.Judgement + if judged == "" { + judged = "none" + } else if w.Judged != "" { + words = strings.TrimSpace(words + " " + w.Judged) + } + out = append(out, []string{w.Node, w.Module, want, have, w.State, judged, words}) } return out } +// sourceText is the source revision's derived state in one line, then one line for each module on a node it waits for. +func sourceText(s Snapshot) []string { + if s.Source == nil { + return nil + } + out := []string{fmt.Sprintf("source revision %d: %s", s.Source.Revision, s.Source.State)} + return append(out, s.Source.Waiting...) +} + // backlogText is one line per language and kind: its waiting asks, the asks held by a node, the oldest waiting ask's age. func backlogText(s Snapshot) [][]string { var out [][]string @@ -254,7 +263,7 @@ func backlogText(s Snapshot) [][]string { var backlogHead = []string{"LANGUAGE", "KIND", "WAITING", "HELD", "OLDEST WAITING"} -var differenceHead = []string{"NODE", "MODULE", "WANT", "SENT", "RULE", "WORDS"} +var wantHead = []string{"NODE", "MODULE", "WANT", "HAVE", "STATE", "MOVE JUDGED", "WORDS"} var head = []string{"NODE", "MODULE", "ASSIGNED", "HAVE", "PHASE", "HEALTH", "SINCE", "DETAIL", "WAITS"} @@ -303,10 +312,16 @@ func Table(w io.Writer, s Snapshot) { row(cells) } } - if d := differenceText(s); len(d) > 0 { + if src := sourceText(s); len(src) > 0 { fmt.Fprintln(tw) - row([]string{"WANTS AGAINST SENDS"}) - row(differenceHead) + for _, line := range src { + row([]string{line}) + } + } + if d := wantText(s); len(d) > 0 { + fmt.Fprintln(tw) + row([]string{"WANTS AGAINST HAVES"}) + row(wantHead) for _, cells := range d { row(cells) } @@ -356,16 +371,16 @@ var page = template.Must(template.New("p").Parse(`CANDIDATES{{range .Groups}}{{range .}}{{end}}{{end}}
GROUPPULL REQUESTHEADMODULES
{{.}}
{{end}} {{if .Asks}}

ASKS

{{range .Asks}}{{range .Cells}}{{end}}{{end}}
ASKHEADLINEWORDSPULL REQUESTSANSWER
{{.}}{{if .Options}}
{{range .Options}} {{end}}
{{end}}
{{end}} {{if .Backlog}}

BACKLOG OF BUILD ASKS

{{range .BacklogHead}}{{end}}{{range .Backlog}}{{range .}}{{end}}{{end}}
{{.}}
{{.}}
{{end}} -{{if .Differences}}

WANTS AGAINST SENDS

{{range .DifferenceHead}}{{end}}{{range .Differences}}{{range .}}{{end}}{{end}}
{{.}}
{{.}}
{{end}} +{{if .Source}}

SOURCE REVISION

{{range .Source}}{{.}}
{{end}}

{{end}}{{if .Wants}}

WANTS AGAINST HAVES

{{range .WantHead}}{{end}}{{range .Wants}}{{range .}}{{end}}{{end}}
{{.}}
{{.}}
{{end}} {{if .Fingerprints}}

FINGERPRINTS

{{range .Fingerprints}}{{range .}}{{end}}{{end}}
MODULESTATEINDEXER AGAINST BUILD
{{.}}
{{end}} {{if .Offline}}

OFFLINE

{{range .Offline}}{{range .}}{{end}}{{end}}
{{.}}
{{end}} {{if .Stuck}}

STUCK

{{range .Stuck}}{{range .}}{{end}}{{end}}
BUCKETKEYWAITED FORSINCE
{{.}}
{{end}} {{if .Unreadable}}

UNREADABLE

{{range .Unreadable}}{{range .}}{{end}}{{end}}
{{.}}
{{end}}`)) var wantsPage = template.Must(template.New("w").Parse(` -Wants against sends -

What each node runs

WANTS AGAINST SENDS

-{{if .Differences}}{{range .Head}}{{end}}{{range .Differences}}{{range .}}{{end}}{{end}}
{{.}}
{{.}}
+Wants against haves +

What each node runs

WANTS AGAINST HAVES

+{{if .Wants}}{{range .Head}}{{end}}{{range .Wants}}{{range .}}{{end}}{{end}}
{{.}}
{{.}}
{{else}}

no difference: every want is what the controller sends.

{{end}}`)) // Web is the read-only web view of the same rows, refreshed by the browser. @@ -387,7 +402,7 @@ func Web(m *Model) http.Handler { } if r.URL.Path == "/wants" { w.Header().Set("Content-Type", "text/html; charset=utf-8") - wantsPage.Execute(w, map[string]any{"Head": differenceHead, "Differences": differenceText(m.Snapshot())}) + wantsPage.Execute(w, map[string]any{"Head": wantHead, "Wants": wantText(m.Snapshot())}) return } if r.URL.Path != "/" { @@ -410,7 +425,7 @@ func Web(m *Model) http.Handler { for _, u := range snap.Unreadable { bad = append(bad, []string{u.Bucket, u.Key, u.Error}) } - page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Differences": differenceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "DifferenceHead": differenceHead, "Offline": off, "Stuck": stuck, "Unreadable": bad}) + page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Wants": wantText(snap), "Source": sourceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "WantHead": wantHead, "Offline": off, "Stuck": stuck, "Unreadable": bad}) }) } diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index dfc0db2..689532f 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -21,6 +21,8 @@ type Entry struct { Value []byte At time.Time Deleted bool + // Revision is the bucket revision of the change, when the watch gives one (the source revision is the gate's source key's). + Revision uint64 } // Row is one module on one node. @@ -66,27 +68,59 @@ type Snapshot struct { Fingerprints []FingerprintRow `json:"fingerprints"` // Asks is every ask the gate holds that has no answer yet: what waits for the operator's Approve. Asks []AskRow `json:"asks"` - // Differences is every difference between what the assigner wants a node to run and what the controller sends, with the rule that - // explains it or unexplained (novox/hq ADR 0528 clause 4, issue 1297 B8). - Differences []DifferenceRow `json:"differences"` + // Wants is every want of the assigner beside the have of its node and where it stands (novox/hq ADR 0539 §8, issue 1906 D7). + Wants []WantRow `json:"wants"` + // Source is the state of the mesh's source revision, derived from the wants and the haves (ADR 0429 §1): recorded by no part. + // Nil when the gate holds no source yet. + Source *SourceRow `json:"source,omitempty"` // Backlog is, for each language and kind of build ask with one waiting or held, the asks waiting for a node, the asks a node // holds and the oldest waiting ask's age (issue 1887 A8a). Backlog []BacklogRow `json:"backlog"` } -// DifferenceRow is a module on a node whose want and send differ: the fingerprint wanted (empty when the node has no want, -// WantRemoved when the want is removed), the fingerprint of the build the controller's assignment names (empty when it sends -// none; SentBuild names that build), and the rule that explains it (Rule 1 to 6) or Unexplained. -type DifferenceRow struct { - Node string `json:"node"` - Module string `json:"module"` - Want string `json:"want,omitempty"` - WantRemoved bool `json:"want-removed,omitempty"` - Sent string `json:"sent,omitempty"` - SentBuild string `json:"sent-build,omitempty"` - Rule int `json:"rule,omitempty"` - Unexplained bool `json:"unexplained,omitempty"` - Words string `json:"words,omitempty"` +// The states of a want against the have of its node. +const ( + WantApplied = "applied" // the node runs the want's fingerprint, running and healthy + WantPending = "pending" // the node has not yet run the want's fingerprint, or not healthy yet + WantFailed = "failed" // the node runs the want's fingerprint and its health check failed + WantHeldBack = "held back" // the assigner holds the node back from the newest fingerprint (Rule), or set it back + WantRemoved = "removed" // the node's want is removed: the module is no longer assigned to it +) + +// WantRow is a want of the assigner beside the have of its node: the fingerprint wanted, the one the node runs with its phase and +// health, the state of the two, and for a want held back the newest fingerprint, the rule and the assigner's words. +type WantRow struct { + Node string `json:"node"` + Module string `json:"module"` + Want string `json:"want,omitempty"` + Have string `json:"have,omitempty"` + Phase string `json:"phase,omitempty"` + Health string `json:"health,omitempty"` + State string `json:"state"` + Newest string `json:"newest,omitempty"` + Rule int `json:"rule,omitempty"` + Words string `json:"words,omitempty"` + // Judgement is the controller's judgement of the move to the composition the want names (recreates, moves-data, neither or + // cannot-judge), and Judged its words; empty when the controller wrote none (ADR 0539 §14). + Judgement string `json:"judgement,omitempty"` + Judged string `json:"judged,omitempty"` +} + +// The states of a source revision (ADR 0429 §1). +const ( + SourceDelivered = "delivered" + SourceNotDelivered = "not delivered" +) + +// SourceRow is the mesh's source revision (the revision of the gate's source key) and whether it is delivered: every want equals its +// module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is assigned to +// has a want. Waiting names each module on a node that is not there yet, and why. +type SourceRow struct { + Revision uint64 `json:"revision"` + State string `json:"state"` + Waiting []string `json:"waiting,omitempty"` + // Wants is how many wants of modules at the indexer were looked at; a revision with none is delivered of nothing. + Wants int `json:"wants"` } // AskRow is an open ask (novox/hq ADR 0437 §10): the gate's ask with no answer under its key in the teller's answers. @@ -178,22 +212,29 @@ type UnreadableRow struct { // Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values. type Model struct { - mu sync.Mutex - assigns map[string]assignment - builds map[string]buildEntry // by .build. - groups map[string]groupEntry // the indexer's candidates, by group - modules map[string]moduleEntry // the indexer's modules, by module - keys map[string]keyEntry // the builder's build keys, by . - asks map[string]askRecord // the gate's asks, by key - diffs map[string]diffEntry // the assigner's differences, by . - aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key - chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key - writer Writer - backlog *Backlog - answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable) - newest map[string]string // module -> id - haves map[string]haveEntry - nodes map[string]nodeEntry + mu sync.Mutex + assigns map[string]assignment + builds map[string]buildEntry // by .build. + groups map[string]groupEntry // the indexer's candidates, by group + modules map[string]moduleEntry // the indexer's modules, by module + keys map[string]keyEntry // the builder's build keys, by . + asks map[string]askRecord // the gate's asks, by key + wants map[string]wantEntry // the assigner's wants, by . + source *sourceEntry // the gate's source key + deliveredEvent func(DeliveredEvent) + motionEvent func(MotionEvent) + told map[string]bool // the motion events said, by their ids + initial map[string]bool // the buckets whose watch handed over what stood at its start + said uint64 // the source revision whose delivery was said + composed map[string]composedEntry // the controller's digests and judgements, by .. + aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key + chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key + writer Writer + backlog *Backlog + answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable) + newest map[string]string // module -> id + haves map[string]haveEntry + nodes map[string]nodeEntry // the watchman's buckets, by key presence map[string]presenceEntry stuck map[string]stuckEntry @@ -256,9 +297,22 @@ type keyEntry struct { err string } -type diffEntry struct { - value delivery.Difference +type wantEntry struct { + value delivery.Want err string + at time.Time +} + +type composedEntry struct { + value delivery.ComposedDigest + err string +} + +type sourceEntry struct { + revision uint64 + group string + heads map[string]string + err string } type assignerAskRecord struct { @@ -290,18 +344,64 @@ type nodeEntry struct { // New is an empty model. func New() *Model { - return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, diffs: map[string]diffEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{}, + return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{}, haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{}, presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}} } -// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed. +// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed. When the change makes +// the source revision delivered, the event of it is said once (SetDeliveredEvent), after the model's lock is released. func (m *Model) Apply(e Entry) { + m.mu.Lock() + m.apply(e) + event := m.deliveredNow() + say := m.deliveredEvent + motion, sayMotion := m.motionNow(e), m.motionEvent + m.mu.Unlock() + if event != nil && say != nil { + say(*event) + } + if sayMotion != nil { + for _, ev := range motion { + sayMotion(ev) + } + } +} + +// DeliveredEvent is what the view says when the source revision is delivered (event `delivered` of the view, issue 1906 D9, ADR +// 0539 §17): the revision of the gate's source key, the group that moved it and every repository's merged head at it. A +// consumer takes every earlier revision as covered by it, and takes the same revision twice as once (after a restart of the view). +type DeliveredEvent struct { + Revision uint64 `json:"revision"` + Group string `json:"group"` + Heads map[string]string `json:"heads"` +} + +// SetDeliveredEvent sets where the event of a delivered source revision goes; nothing is said until it is set. +func (m *Model) SetDeliveredEvent(say func(DeliveredEvent)) { m.mu.Lock() defer m.mu.Unlock() + m.deliveredEvent = say +} + +// deliveredNow is the event of the source revision when it is delivered now and has not been said, or nil. A revision that went +// bad after it was delivered and recovered is not said again. +func (m *Model) deliveredNow() *DeliveredEvent { + if m.deliveredEvent == nil || m.source == nil || m.source.err != "" || m.said == m.source.revision { + return nil + } + row := m.sourceRow() + if row == nil || row.State != SourceDelivered || row.Wants == 0 { + return nil + } + m.said = m.source.revision + return &DeliveredEvent{Revision: m.source.revision, Group: m.source.group, Heads: m.source.heads} +} + +func (m *Model) apply(e Entry) { switch e.Bucket { case delivery.HaveBucket: - node, module, isNode, err := delivery.ParseHaveKey(e.Key) + node, _, isNode, err := delivery.ParseHaveKey(e.Key) if err != nil { return } @@ -314,7 +414,7 @@ func (m *Model) Apply(e Entry) { m.nodes[node] = nodeEntry{node: n, err: errText(err), at: e.At} return } - key := node + "." + module + key := e.Key // the bucket's own key: a module name with a dot is escaped in it (issue 1150) if e.Deleted { delete(m.haves, key) return @@ -385,13 +485,38 @@ func (m *Model) Apply(e Entry) { if a, err := delivery.DecodeViewAnswer(e.Value); err == nil { m.chosen[askKey] = a } - case delivery.AssignerDifferencesBucket: - if e.Deleted { - delete(m.diffs, e.Key) + case delivery.AssignerWantsBucket: + node, module, err := delivery.ParseWantKey(e.Key) + if err != nil { + return // a hold, and the node's own key, are not a module's want + } + key, kerr := delivery.HaveKey(node, module) + if kerr != nil { return } - d, err := delivery.DecodeDifference(e.Value) - m.diffs[e.Key] = diffEntry{value: d, err: errText(err)} + if e.Deleted { + delete(m.wants, key) + return + } + w, err := delivery.DecodeWant(e.Value) + m.wants[key] = wantEntry{value: w, err: errText(err), at: e.At} + case delivery.ControllerComposedBucket: + if e.Deleted { + delete(m.composed, e.Key) + return + } + d, err := delivery.DecodeComposedDigest(e.Value) + m.composed[e.Key] = composedEntry{value: d, err: errText(err)} + case delivery.SourceBucket: + if e.Key != delivery.SourceMainKey { + return + } + if e.Deleted { + m.source = nil + return + } + src, err := delivery.DecodeSource(e.Value) + m.source = &sourceEntry{revision: e.Revision, group: src.Group, heads: src.Heads, err: errText(err)} case delivery.AnswersBucket: if e.Deleted { delete(m.answers, e.Key) @@ -487,7 +612,17 @@ func (m *Model) snapshot() Snapshot { r.RequiredBy = a.RequiredBy } var waits []string - h, has := m.haves[key] + hk, herr := delivery.HaveKey(a.Node, a.Module) + if herr != nil { + hk = key + } + h, has := m.haves[hk] + // The controller no longer writes the build of an assignment (R6): the row is the want and the have of the node. + w, wanted := m.wants[hk] + wanted = wanted && w.err == "" && !w.value.Removed && w.value.FP != "" + if wanted { + r.Assigned = w.value.FP + } switch { case !has: waits = append(waits, "no have yet") @@ -495,8 +630,15 @@ func (m *Model) snapshot() Snapshot { r.Since, r.Error = h.at, h.err default: r.Have, r.Phase, r.Health = h.have.Build, string(h.have.Phase), string(h.have.Health) + if r.Have == "" { + r.Have = h.have.FP + } r.Since, r.Detail = h.at, h.have.Detail - if a.Build != "" && h.have.Build != a.Build { + if wanted { + if h.have.FP != w.value.FP { + waits = append(waits, "the node has not applied "+w.value.FP) + } + } else if a.Build != "" && h.have.Build != a.Build { waits = append(waits, "the node has not applied "+a.Build) } } @@ -509,10 +651,13 @@ func (m *Model) snapshot() Snapshot { rows = append(rows, r) } for key, h := range m.haves { - if a, ok := m.assigns[key]; ok && a.open() { + node, module, _, perr := delivery.ParseHaveKey(key) + if perr != nil { + continue + } + if a, ok := m.assigns[node+"."+module]; ok && a.open() { continue } - node, module, _, _ := delivery.ParseHaveKey(key) r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err} if h.err == "" && h.have.Phase == delivery.PhaseRemoved { // The node-engine says it took the module away: plainly removed, not left behind. @@ -520,6 +665,9 @@ func (m *Model) snapshot() Snapshot { } if h.err == "" { r.Have, r.Phase, r.Health, r.Detail = h.have.Build, string(h.have.Phase), string(h.have.Health), h.have.Detail + if r.Have == "" { + r.Have = h.have.FP + } } rows = append(rows, r) } @@ -585,9 +733,9 @@ func (m *Model) snapshot() Snapshot { unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerAsksBucket, Key: key, Error: a.err}) } } - for key, d := range m.diffs { - if d.err != "" { - unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerDifferencesBucket, Key: key, Error: d.err}) + for key, w := range m.wants { + if w.err != "" { + unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerWantsBucket, Key: key, Error: w.err}) } } sort.Slice(offline, func(i, j int) bool { @@ -601,7 +749,7 @@ func (m *Model) snapshot() Snapshot { return unreadable[i].Bucket+"."+unreadable[i].Key < unreadable[j].Bucket+"."+unreadable[j].Key }) return Snapshot{Rows: rows, Nodes: nodes, Offline: offline, Stuck: stuck, Unreadable: unreadable, - Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Differences: m.differenceRows()} + Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Wants: m.wantRows(), Source: m.sourceRow()} } // groupRows are the readable candidate groups, ordered by group, their pull requests by repository and number. @@ -840,20 +988,38 @@ func (m *Model) Answer(askKey, option, words string) error { return w.Put(key, body) } -// differenceRows are the readable differences, ordered by node and module. -func (m *Model) differenceRows() []DifferenceRow { - out := []DifferenceRow{} - for key, d := range m.diffs { - if d.err != "" { +// wantRows are the readable wants beside the haves of their nodes, ordered by node and module. +func (m *Model) wantRows() []WantRow { + out := []WantRow{} + for key, w := range m.wants { + if w.err != "" { continue } - node, module, err := delivery.ParseDifferenceKey(key) - if err != nil { - continue + node, module := splitKey(key) + v := w.value + row := WantRow{Node: node, Module: module, Want: v.FP, Newest: v.Newest, Rule: v.Rule, Words: v.Words} + h, has := m.haves[key] + if has && h.err == "" { + row.Have, row.Phase, row.Health = h.have.FP, string(h.have.Phase), string(h.have.Health) } - v := d.value - out = append(out, DifferenceRow{Node: node, Module: module, Want: v.Want, WantRemoved: v.WantRemoved, Sent: v.Sent, SentBuild: v.SentBuild, - Rule: v.Rule, Unexplained: v.Unexplained, Words: v.Words}) + if ckey, err := delivery.ComposedKey(node, module, v.FP); err == nil && !v.Removed { + if c, ok := m.composed[ckey]; ok && c.err == "" { + row.Judgement, row.Judged = string(c.value.Judgement), c.value.Words + } + } + switch { + case v.Removed: + row.State = WantRemoved + case v.Rule != 0 || v.Newest != "": + row.State = WantHeldBack + case row.Have == v.FP && h.have.Health == delivery.HealthFailed: + row.State = WantFailed + case row.Have == v.FP && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK: + row.State = WantApplied + default: + row.State = WantPending + } + out = append(out, row) } sort.Slice(out, func(i, j int) bool { if out[i].Node != out[j].Node { @@ -863,3 +1029,257 @@ func (m *Model) differenceRows() []DifferenceRow { }) return out } + +// sourceRow derives the state of the source revision the gate holds (ADR 0429 §1; recorded by no part): delivered when every want +// equals its module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is +// assigned to has a want. It is nil when the gate holds no readable source. +func (m *Model) sourceRow() *SourceRow { + if m.source == nil || m.source.err != "" { + return nil + } + waiting := []string{} + short := func(fp string) string { + fp = strings.TrimPrefix(fp, delivery.SourcePrefix2) + if len(fp) > 12 { + fp = fp[:12] + } + return fp + } + looked := 0 + for key, w := range m.wants { + node, module := splitKey(key) + mod, ok := m.modules[module] + if w.err != "" || w.value.Removed || !ok || mod.err != "" || mod.value.FP2 == "" { + continue + } + looked++ + h := m.haves[key] + switch { + case w.value.FP != mod.value.FP2: + waiting = append(waiting, fmt.Sprintf("%s on %s wants %s, not the module's %s", module, node, short(w.value.FP), short(mod.value.FP2))) + case h.err != "" || h.have.FP != w.value.FP: + waiting = append(waiting, fmt.Sprintf("%s on %s does not yet run %s", module, node, short(w.value.FP))) + case h.have.Health == delivery.HealthFailed: + waiting = append(waiting, fmt.Sprintf("%s on %s runs %s and its health check failed", module, node, short(w.value.FP))) + case h.have.Phase != delivery.PhaseRunning || h.have.Health != delivery.HealthOK: + waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet running and healthy", module, node, short(w.value.FP))) + } + } + for key, a := range m.assigns { + if !a.open() { + continue + } + wk, werr := delivery.HaveKey(a.Node, a.Module) + if werr != nil { + wk = key + } + if _, ok := m.wants[wk]; ok { + continue + } + if mod, ok := m.modules[a.Module]; ok && mod.err == "" && mod.value.FP2 != "" { + waiting = append(waiting, fmt.Sprintf("%s on %s has no want yet", a.Module, a.Node)) + } + } + sort.Strings(waiting) + row := &SourceRow{Revision: m.source.revision, State: SourceDelivered, Wants: looked} + if len(waiting) > 0 { + row.State, row.Waiting = SourceNotDelivered, waiting + } + return row +} + +// splitKey is the node and the module (unescaped, issue 1150) of a model key, which is the have bucket's key form. +func splitKey(key string) (node, module string) { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + node, module, _ = strings.Cut(key, ".") + } + return node, module +} + +// The events of a module in motion (issue 1906 D10, for the messenger, ADR 0308): with no plan to say it, the view says that a +// module's want on a node differs from its have (moving), that the have reached the want running and healthy (settled), or that it +// is held or failed or stopped moving past the watchman's bound (stuck). Each is said once for a node, a module and a composition. +const ( + MotionMoving = "moving" + MotionSettled = "settled" + MotionStuck = "stuck" + // MotionBoundSeconds is how long a node may take to apply a want before the watchman says it stuck (watch.StepBound). + MotionBoundSeconds = 1800 +) + +// MotionBody is the body of the three events. Outcome and Words are those of settled and stuck: settled is "running"; stuck is +// "failed" (a health check failed) or "stopped" (the module is held, or the watchman says the want stopped moving past its bound), +// with the have's own words or the watchman's in Words. +type MotionBody struct { + Node string `json:"node"` + Module string `json:"module"` + FromFP string `json:"from_fp"` + ToFP string `json:"to_fp"` + Composed string `json:"composed"` + Since time.Time `json:"since"` + BoundSeconds int `json:"bound_seconds"` + // At is when the have or the want last changed, the time the event is about. + At time.Time `json:"at"` + Outcome string `json:"outcome,omitempty"` + Words string `json:"words,omitempty"` +} + +// MotionEvent is an event to say: its type (moving, settled or stuck), its id (the bus's dedup id and the event id header, one for +// a type, a node, a module and a composition) and its body. +type MotionEvent struct { + Type string + ID string + Body MotionBody +} + +// SetMotionEvent sets where the events of modules in motion go; nothing is said until it is set. +func (m *Model) SetMotionEvent(say func(MotionEvent)) { + m.mu.Lock() + defer m.mu.Unlock() + m.motionEvent = say +} + +// motionNow is the events to say now about the node and module the change touched: a want, a have, or the watchman's stuck key of a +// want. +func (m *Model) motionNow(e Entry) []MotionEvent { + if m.motionEvent == nil || !m.motionReady() { + return nil + } + var hk string + switch e.Bucket { + case delivery.AssignerWantsBucket: + node, module, err := delivery.ParseWantKey(e.Key) + if err != nil { + return nil + } + hk, _ = delivery.HaveKey(node, module) + case delivery.HaveBucket: + if _, _, isNode, err := delivery.ParseHaveKey(e.Key); err != nil || isNode { + return nil + } + hk = e.Key + case delivery.StuckBucket: + st, ok := m.stuck[e.Key] + if !ok || st.err != "" || st.value.Bucket != delivery.AssignerWantsBucket { + return nil + } + node, module, err := delivery.ParseWantKey(st.value.Key) + if err != nil { + return nil + } + hk, _ = delivery.HaveKey(node, module) + default: + return nil + } + return m.motionOf(hk) +} + +// motionReady says whether the view has read what stands in the buckets motion is derived from (the wants, the haves and the +// watchman's stuck keys): before, a want read ahead of its have would be said moving though the node runs it. +func (m *Model) motionReady() bool { + for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} { + if !m.initial[b] { + return false + } + } + return true +} + +// Initial tells the model that the watch of a bucket has handed over every key that stood at its start. When the buckets motion is +// derived from have all done so, the events of the modules in motion at that moment are said (the ids deduplicate what a restart +// of the view says again). +func (m *Model) Initial(bucket string) { + m.mu.Lock() + was := m.motionReady() + m.initial[bucket] = true + var out []MotionEvent + if !was && m.motionReady() && m.motionEvent != nil { + keys := make([]string, 0, len(m.wants)) + for k := range m.wants { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + out = append(out, m.motionOf(k)...) + } + } + say := m.motionEvent + m.mu.Unlock() + for _, ev := range out { + say(ev) + } +} + +// motionOf is the events to say now about the node and module of a model key, from their want, have and stuck key. +func (m *Model) motionOf(hk string) []MotionEvent { + w, ok := m.wants[hk] + if !ok || w.err != "" || w.value.Removed || w.value.FP == "" { + return nil + } + node, module := splitKey(hk) + h, has := m.haves[hk] + has = has && h.err == "" + body := MotionBody{Node: node, Module: module, ToFP: w.value.FP, Composed: w.value.Composed, Since: w.at, BoundSeconds: MotionBoundSeconds} + body.At = w.at + if has { + body.FromFP = h.have.FP + if h.at.After(body.At) { + body.At = h.at + } + } + id := func(kind string) string { + return kind + "." + node + "." + module + "." + strings.TrimPrefix(w.value.Composed, "sha256:") + } + applied := has && h.have.FP == w.value.FP && (h.have.Composed == "" || h.have.Composed == w.value.Composed) + var out []MotionEvent + say := func(kind string, b MotionBody) { + if m.told[id(kind)] { + return + } + m.told[id(kind)] = true + out = append(out, MotionEvent{Type: kind, ID: id(kind), Body: b}) + } + switch { + case applied && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK: + if m.told[id(MotionMoving)] { + b := body + b.Outcome = "running" + say(MotionSettled, b) + } + return out + case !m.told[id(MotionMoving)]: + say(MotionMoving, body) + } + stop := body + switch { + case has && h.have.Phase == delivery.PhaseHeld: + stop.Outcome, stop.Words = "stopped", h.have.Detail + case has && (h.have.Health == delivery.HealthFailed || h.have.Phase == delivery.PhaseFailed) && h.have.FP == w.value.FP: + stop.Outcome, stop.Words = "failed", h.have.Detail + default: + if s, ok := m.stuckWant(w.value, node, module); ok { + stop.Outcome, stop.Words = "stopped", s + } else { + return out + } + } + say(MotionStuck, stop) + return out +} + +// stuckWant is the watchman's words for the want of module on node, when it says the want stopped moving and not yet moved. +func (m *Model) stuckWant(_ delivery.Want, node, module string) (string, bool) { + wk, err := delivery.WantKey(node, module) + if err != nil { + return "", false + } + sk, err := delivery.StuckKey(delivery.AssignerWantsBucket, wk) + if err != nil { + return "", false + } + if st, ok := m.stuck[sk]; ok && st.err == "" && !st.value.Moved { + return st.value.Waited, true + } + return "", false +} diff --git a/modules/mesh-delivery-view/internal/view/wants_test.go b/modules/mesh-delivery-view/internal/view/wants_test.go new file mode 100644 index 0000000..42832ef --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/wants_test.go @@ -0,0 +1,348 @@ +package view + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const ( + fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111" + fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222" +) + +func wantE(t *testing.T, node, module string, w delivery.Want) Entry { + t.Helper() + key, err := delivery.WantKey(node, module) + if err != nil { + t.Fatal(err) + } + if !w.Removed { + w.Composed = "sha256:" + strings.Repeat("c", 64) + } + v, err := delivery.EncodeWant(w) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.AssignerWantsBucket, Key: key, Value: v, At: t0} +} + +func haveFP(t *testing.T, node, module, fp string, health delivery.Health) Entry { + t.Helper() + b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: health}) + if err != nil { + t.Fatal(err) + } + key, err := delivery.HaveKey(node, module) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.HaveBucket, Key: key, Value: b, At: t0} +} + +func moduleE(t *testing.T, module, fp string) Entry { + t.Helper() + v, err := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/" + module, Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.ModulesBucket, Key: module, Value: v, At: t0} +} + +func sourceE(t *testing.T, revision uint64) Entry { + t.Helper() + v, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) + if err != nil { + t.Fatal(err) + } + return Entry{Bucket: delivery.SourceBucket, Key: delivery.SourceMainKey, Value: v, At: t0, Revision: revision} +} + +// 1906/D7 AC7: every want is a row beside the have of its node, with where it stands. +func TestEveryWantIsARowAgainstItsHave(t *testing.T) { + m := New() + fill(m, + wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK), + wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK), + wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed), + wantE(t, "g14", "mail", delivery.Want{FP: fpOld, Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"}), + wantE(t, "g14", "old", delivery.Want{Removed: true}), + wantE(t, "novox", "web", delivery.Want{FP: fpNew}), + ) + got := map[string]WantRow{} + for _, r := range m.Snapshot().Wants { + got[r.Node+"."+r.Module] = r + } + for key, state := range map[string]string{ + "ace.web": WantApplied, "ace.mail": WantPending, "g14.web": WantFailed, "g14.mail": WantHeldBack, "g14.old": WantRemoved, "novox.web": WantPending, + } { + if got[key].State != state { + t.Errorf("%s = %+v, want state %s", key, got[key], state) + } + } + if r := got["g14.mail"]; r.Rule != delivery.RuleCanary || r.Newest != fpNew || r.Words == "" { + t.Errorf("the held-back row keeps its rule and words: %+v", r) + } + if r := got["ace.mail"]; r.Want != fpNew || r.Have != fpOld { + t.Errorf("a pending row shows both: %+v", r) + } + // a want taken away leaves the list + fill(m, Entry{Bucket: delivery.AssignerWantsBucket, Key: "want.novox.web", Deleted: true}) + if n := len(m.Snapshot().Wants); n != 5 { + t.Errorf("wants = %d after a delete, want 5", n) + } +} + +// 1906/D7 AC7: a source revision is delivered when every want equals its module's fingerprint and every have equals its +// want, healthy; one failed have shows it not delivered and names the module. +func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) { + m := New() + fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew), + assign("ace", "web", "b-1", ""), assign("g14", "web", "b-1", ""), + wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK), + wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK), + wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpNew, delivery.HealthOK)) + s := m.Snapshot().Source + if s == nil || s.Revision != 14355 || s.State != SourceDelivered || len(s.Waiting) != 0 { + t.Fatalf("source = %+v, want revision 14355 delivered", s) + } + fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthFailed)) + s = m.Snapshot().Source + if s.State != SourceNotDelivered || len(s.Waiting) != 1 || !strings.Contains(s.Waiting[0], "web") || !strings.Contains(s.Waiting[0], "g14") { + t.Errorf("a failed have: %+v", s) + } + fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "mail", delivery.Want{FP: fpOld})) + s = m.Snapshot().Source + if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") { + t.Errorf("a want that is not the module's fingerprint: %+v", s) + } + fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), assign("novox", "web", "b-1", "")) + s = m.Snapshot().Source + if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") { + t.Errorf("a node assigned the module with no want: %+v", s) + } +} + +func TestNoSourceRevisionHeldMeansNoStateIsDerived(t *testing.T) { + if s := New().Snapshot().Source; s != nil { + t.Errorf("source = %+v", s) + } +} + +func TestTheWantsPageSaysWantsAgainstHaves(t *testing.T) { + m := New() + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) + text := wantText(m.Snapshot()) + if len(text) != 1 || text[0][0] != "ace" || text[0][1] != "web" || text[0][2] != fpNew || text[0][3] != fpOld || text[0][4] != WantPending || text[0][5] != "none" { + t.Errorf("text = %v", text) + } +} + +// 1906/D7 (ADR 0539 §14): the controller's judgement of the move a want names is shown beside it, with its words. +func TestTheJudgementOfTheMoveIsShownBesideTheWant(t *testing.T) { + m := New() + key, err := delivery.ComposedKey("ace", "web", fpNew) + if err != nil { + t.Fatal(err) + } + v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}) + if err != nil { + t.Fatal(err) + } + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.ControllerComposedBucket, Key: key, Value: v, At: t0}) + rows := m.Snapshot().Wants + if len(rows) != 1 || rows[0].Judgement != string(delivery.JudgedCannot) || !strings.Contains(rows[0].Judged, "gone") { + t.Fatalf("rows = %+v", rows) + } + text := wantText(m.Snapshot()) + if len(text[0]) != 7 || text[0][5] != "cannot-judge" { + t.Errorf("text = %v", text) + } +} + +// 1906/D9 (asked by the lead for issue 1946): the view says, once for each source revision, that it is delivered: the event the +// issue tracker derives "fix delivered" from, in place of the gate's states that go in release 6 (ADR 0539 §17). +func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) { + m := New() + var said []DeliveredEvent + m.SetDeliveredEvent(func(e DeliveredEvent) { said = append(said, e) }) + fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), assign("ace", "web", "b-1", ""), + wantE(t, "ace", "web", delivery.Want{FP: fpNew})) + if len(said) != 0 { + t.Fatalf("said %v before any have", said) + } + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed)) + if len(said) != 0 { + t.Fatalf("said %v for a failed have", said) + } + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK)) + if len(said) != 1 || said[0].Revision != 14355 || said[0].Group != "g" || said[0].Heads["novox/web"] == "" { + t.Fatalf("said = %+v, want revision 14355 once with its group and heads", said) + } + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew})) + if len(said) != 1 { + t.Errorf("said again for the same revision: %v", said) + } + // a newer revision is delivered when its wants and haves are; a revision that goes bad and good again is not said twice + fill(m, sourceE(t, 14356)) + if len(said) != 2 || said[1].Revision != 14356 { + t.Errorf("the next revision: %v", said) + } + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed), haveFP(t, "ace", "web", fpNew, delivery.HealthOK)) + if len(said) != 2 { + t.Errorf("said again after a have failed and recovered: %v", said) + } +} + +// Review round 1 (view notes): the controller no longer writes the build of an assignment (C6), so a row reads the want and the +// have: the row shows the fingerprint wanted and waits while the node runs another. +func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) { + m := New() + fill(m, assign("ace", "web", "", ""), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) + r := find(t, m.Snapshot(), "ace", "web") + if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") { + t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r) + } + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK)) + if r := find(t, m.Snapshot(), "ace", "web"); r.Waits != "" || r.Have != fpNew { + t.Errorf("row = %+v, want no wait once the have names the want's fingerprint", r) + } +} + +// Review round 1 (view notes, issue 1150): a module whose name holds a dot has its have under the escaped key; the row finds it, +// and a have nothing is assigned to keeps the module's name. +func TestADottedModuleNameIsFoundAndKept(t *testing.T) { + m := New() + fill(m, assign("ace", "novox.be", "", ""), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK), + haveFP(t, "ace", "old.name", fpNew, delivery.HealthOK)) + s := m.Snapshot() + if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Waits != "" { + t.Errorf("the dotted module's row = %+v", r) + } + if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind { + t.Errorf("the left-behind dotted module = %+v", r) + } + if rows := s.Wants; len(rows) != 1 || rows[0].Module != "novox.be" || rows[0].State != WantApplied { + t.Errorf("wants = %+v", rows) + } +} + +// 1906/D10 (asked by the lead for the messenger, ADR 0308): with no plans to say that a module is in motion, the view says it: `moving` +// when a module's want on a node differs from its have, `settled` when the have reaches the want running and healthy, `stuck` when it +// is held or failed or the watchman says it stopped moving; each once for a node, a module and a composition. +func motionLog(m *Model) *[]MotionEvent { + var said []MotionEvent + m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) }) + for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} { + m.Initial(b) + } + return &said +} + +func types(said []MotionEvent) string { + var out []string + for _, e := range said { + out = append(out, e.Type) + } + return strings.Join(out, ",") +} + +func TestAModuleInMotionIsSaidMovingThenSettledOnceEach(t *testing.T) { + m := New() + said := motionLog(m) + fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) + if len(*said) != 0 { + t.Fatalf("said %v for a have with no want", types(*said)) + } + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew})) + if types(*said) != "moving" { + t.Fatalf("said %q, want moving", types(*said)) + } + mv := (*said)[0] + if mv.Body.Node != "ace" || mv.Body.Module != "web" || mv.Body.FromFP != fpOld || mv.Body.ToFP != fpNew || mv.Body.Composed != "sha256:"+strings.Repeat("c", 64) || + mv.Body.Since.IsZero() || mv.Body.At.IsZero() || mv.Body.BoundSeconds != 1800 || mv.ID != "moving.ace.web."+strings.Repeat("c", 64) { + t.Errorf("moving = %+v", mv) + } + fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew})) + if types(*said) != "moving" { + t.Errorf("said %q after the same state again", types(*said)) + } + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK)) + if types(*said) != "moving,settled" || (*said)[1].Body.Outcome != "running" { + t.Errorf("said %q, want settled with its outcome", types(*said)) + } + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK)) + if types(*said) != "moving,settled" { + t.Errorf("settled said twice: %q", types(*said)) + } +} + +func TestAModuleHeldOrFailedIsSaidStuckWithItsWords(t *testing.T) { + m := New() + said := motionLog(m) + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew})) + b, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"}) + fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0}) + if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" || (*said)[1].Body.Words != "no space left" { + t.Fatalf("said %v: %+v", types(*said), *said) + } + fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0}) + if types(*said) != "moving,stuck" { + t.Errorf("stuck said twice: %q", types(*said)) + } + // failed on the wanted fingerprint + m2 := New() + said2 := motionLog(m2) + fill(m2, wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed)) + if types(*said2) != "moving,stuck" || (*said2)[1].Body.Outcome != "failed" { + t.Errorf("failed: said %q", types(*said2)) + } +} + +func TestAWantTheWatchmanSaysStoppedMovingIsSaidStuckPastItsBound(t *testing.T) { + m := New() + said := motionLog(m) + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK)) + sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, "want.ace.web") + v, err := delivery.EncodeStuck(delivery.Stuck{Bucket: delivery.AssignerWantsBucket, Key: "want.ace.web", Waited: "the node agent of ace to run web", Since: t0}) + if err != nil { + t.Fatal(err) + } + fill(m, Entry{Bucket: delivery.StuckBucket, Key: sk, Value: v, At: t0}) + if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" { + t.Errorf("said %q: %+v", types(*said), *said) + } +} + +func TestANodeThatSettlesWithoutBeingSeenMovingSaysNothing(t *testing.T) { + m := New() + said := motionLog(m) + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew})) + if len(*said) != 0 { + t.Errorf("said %v for a module that never moved", types(*said)) + } +} + +// Nothing is said of a module in motion before the buckets have been read: a want read ahead of its have is not a node behind. What is +// in motion when they have been read is said then, and a module that settles after is said settled. +func TestMotionIsSaidOnlyOnceTheBucketsHaveBeenRead(t *testing.T) { + m := New() + var said []MotionEvent + m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) }) + fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), wantE(t, "ace", "mail", delivery.Want{FP: fpNew})) + m.Initial(delivery.AssignerWantsBucket) + fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK)) + m.Initial(delivery.HaveBucket) + if len(said) != 0 { + t.Fatalf("said %v before the stuck keys were read", types(said)) + } + m.Initial(delivery.StuckBucket) + if len(said) != 1 || said[0].Type != MotionMoving || said[0].Body.Module != "mail" { + t.Fatalf("said %v: %+v, want only mail moving (web is already there)", types(said), said) + } + fill(m, haveFP(t, "ace", "mail", fpNew, delivery.HealthOK)) + if types(said) != "moving,settled" { + t.Errorf("said %q", types(said)) + } +} diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 871bca3..ebee545 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -50,6 +50,7 @@ type manifest struct { Reads []string `json:"reads"` Uses []string `json:"uses"` Tools []string `json:"tools"` + Emits []string `json:"emits"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -83,17 +84,21 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { t.Errorf("reads = %v, want exactly %v", got, want) } - for _, k := range []string{"emits", "consumes", "provides", "claims"} { + for _, k := range []string{"consumes", "provides", "claims"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the view writes only its answers", k) } } + // The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves. + if !reflect.DeepEqual(m.Emits, []string{"delivered", "moving", "settled", "stuck"}) { + t.Errorf("emits = %v, want exactly delivered, moving, settled and stuck (1906/D9, D10)", m.Emits) + } // The view uses no build seat: it reads their workers by the broker's named grant and publishes no ask (ADR 0525 §2). if len(m.Uses) != 0 { t.Errorf("uses = %v, want none", m.Uses) @@ -110,7 +115,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 61ed274..5095b0c 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -9,7 +9,17 @@ "indexer_candidate", "indexer_candidates", "builder_build", - "builder_builds" + "builder_builds", + "assigner_wants", + "assigner_want", + "assigner_differences", + "node_haves" + ], + "emits": [ + "delivered", + "moving", + "settled", + "stuck" ], "state": [ { "name": "answers", "history": 5 } @@ -17,10 +27,13 @@ "reads": [ "mesh-controller.assignments", "mesh-controller.builds", + "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", + "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", + "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index 5d56874..6e99302 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "time" @@ -99,6 +100,8 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 {Bucket: delivery.StuckBucket, History: 20}, {Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second}, {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, + {Bucket: delivery.AssignerWantsBucket, History: 20}, + {Bucket: delivery.HaveBucket, History: 20}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -265,6 +268,8 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 {Bucket: delivery.StuckBucket, History: 20}, {Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second}, {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, + {Bucket: delivery.AssignerWantsBucket, History: 20}, + {Bucket: delivery.HaveBucket, History: 20}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -302,3 +307,134 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 } t.Fatalf("no pulse for the ask running before the start (%s)", pk) } + +// TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt (1906/D6, ADR 0539 §19): started whole, the watchman mirrors a want of +// the assigner as a pulse owed by the node's agent, one that stood before its start as well, and the node-engine's have that +// names the want's fingerprint deletes the pulse and writes no stuck key. +func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "w") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + conf := filepath.Join(dir, "bus.conf") + if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + srv, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(srv.Shutdown) + ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(ctl.Close) + js, err := jetstream.New(ctl) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + kvs := map[string]jetstream.KeyValue{} + for _, c := range []jetstream.KeyValueConfig{ + {Bucket: delivery.PresenceBucket, History: 20}, + {Bucket: delivery.StuckBucket, History: 20}, + {Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second}, + {Bucket: delivery.StepsBucket("mesh-build-go"), History: 5}, + {Bucket: delivery.AssignerWantsBucket, History: 20}, + {Bucket: delivery.HaveBucket, History: 20}, + } { + kv, err := js.CreateKeyValue(ctx, c) + if err != nil { + t.Fatal(err) + } + kvs[c.Bucket] = kv + } + fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64) + fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64) + putWant := func(node, module, fp string) { + t.Helper() + body, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.WantKey(node, module) + if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, key, body); err != nil { + t.Fatal(err) + } + } + putHave := func(node, module, fp string) { + t.Helper() + body, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.HaveKey(node, module) + if _, err := kvs[delivery.HaveBucket].Put(ctx, key, body); err != nil { + t.Fatal(err) + } + } + putHave("n2", "web", fpOld) + putWant("n2", "web", fpNew) // stood before the start + broker := filepath.Join(dir, "broker") + cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-watchman", User: "n1.mesh-delivery-watchman", Password: "pw"}) + if err := os.WriteFile(broker, cred, 0o600); err != nil { + t.Fatal(err) + } + env := map[string]string{"MESH_BROKER_FILE": broker} + runCtx, stop := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() + t.Cleanup(func() { stop(); <-done }) + pulse := func(node, module string) string { + wk, _ := delivery.WantKey(node, module) + pk, err := delivery.PulseKey(delivery.AssignerWantsBucket, wk) + if err != nil { + t.Fatal(err) + } + return pk + } + until := func(what string, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never saw: %s", what) + } + has := func(key string) bool { _, err := kvs[delivery.PulseBucket].Get(ctx, key); return err == nil } + until("a pulse for the want that stood before the start", func() bool { return has(pulse("n2", "web")) }) + putWant("n2", "mail", fpNew) + until("a pulse for a want written after the start", func() bool { return has(pulse("n2", "mail")) }) + e, err := kvs[delivery.PulseBucket].Get(ctx, pulse("n2", "web")) + if err != nil { + t.Fatal(err) + } + if p, err := delivery.DecodePulse(e.Value()); err != nil || !strings.Contains(p.Owes, "n2") { + t.Errorf("the pulse = %+v, %v; want it owed by the agent of n2", p, err) + } + putHave("n2", "web", fpNew) + until("the pulse of the want its have names deleted", func() bool { return !has(pulse("n2", "web")) }) + if !has(pulse("n2", "mail")) { + t.Errorf("the pulse of another want ended with web's have") + } + for _, m := range []string{"web"} { + wk, _ := delivery.WantKey("n2", m) + sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, wk) + if _, err := kvs[delivery.StuckBucket].Get(ctx, sk); err == nil { + t.Errorf("a stuck key was written for the want of %s that was applied", m) + } + } +} diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 9a9be02..f619da4 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -229,11 +229,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return kv, nil } - var kvs struct{ presence, pulse, stuck jetstream.KeyValue } + var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue } for _, b := range []struct { name string into *jetstream.KeyValue - }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} { + }{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}, + {delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} { if *b.into, err = bucket(b.name); err != nil { return err } @@ -280,6 +281,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } + // What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent + // until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse. + haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) + if err != nil { + return err + } + wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) + if err != nil { + return err + } // The build seats' steps: a step key `.` written is the ask taken (issue 1887 A8b). A seat whose build module // keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same. taken := map[string]jetstream.KeyWatcher{} @@ -337,6 +348,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) }) }() + go func() { + errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) + }() + go func() { + errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) + }() for lang, sw := range taken { lang, sw := lang, sw go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }() diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index d0dc67e..d343e0e 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-watchman go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-watchman/go.sum b/modules/mesh-delivery-watchman/go.sum index 8dd6f35..bfba8a6 100644 --- a/modules/mesh-delivery-watchman/go.sum +++ b/modules/mesh-delivery-watchman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index 6711785..78e54c5 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -1,6 +1,9 @@ package watch import ( + "strings" + "time" + "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -52,3 +55,96 @@ func (w *Watchman) Step(language, key string, value []byte, op Op) { } w.mirrorWithin(w.stepBound, bucket, id, describe("the ask %s to end on %s", id, node), node, since, 0) } + +// A want is a move too (novox/hq ADR 0539 §19, issue 1906): the assigner writes `want..` in its wants bucket, +// and the node's agent owes the node-engine's have naming the want's fingerprint. The pulse is the pulse of that bucket under +// the want's key, owed by the node's agent, and it begins when the assigner wrote the want. The have naming the fingerprint +// ends it; a want that is removed, or a key deleted, ends it as well. Nothing renews it: the bound is how long a node may +// take to apply a want, the same as a build step's (StepBound). Keys of that bucket that are not a want of a module (a hold, +// and the node's own key) are left. + +// Want is a change of a key of the assigner's wants bucket, read from the watch (at the start the keys that stand, then each +// change), at the time the entry was written. +func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { + node, module, err := delivery.ParseWantKey(key) + if err != nil { + return + } + w.mu.Lock() + defer w.mu.Unlock() + switch op { + case Expired: + return + case Delete: + delete(w.wants, key) + w.moved(delivery.AssignerWantsBucket, key) + return + } + want, err := delivery.DecodeWant(value) + if err != nil { + w.say("the want %s cannot be read, and is left: %v", key, err) + return + } + if want.Removed { + delete(w.wants, key) + w.moved(delivery.AssignerWantsBucket, key) + return + } + w.wants[key] = want.FP + if w.have(node, module) == want.FP { + w.moved(delivery.AssignerWantsBucket, key) + return + } + if at.IsZero() { + at = w.now() + } + w.mirrorWithin(w.stepBound, delivery.AssignerWantsBucket, key, + describe("the node agent of %s to run %s at %s", node, module, shortFP(want.FP)), want.FP, at, 0) +} + +// Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the +// fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left. +func (w *Watchman) Have(key string, value []byte, op Op) { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + return + } + w.mu.Lock() + defer w.mu.Unlock() + switch op { + case Expired: + return + case Delete: + delete(w.haves, key) + return + } + have, err := delivery.DecodeHave(value) + if err != nil { + w.say("the have %s cannot be read, and is left: %v", key, err) + return + } + w.haves[key] = have.FP + wk, err := delivery.WantKey(node, module) + if err != nil { + return + } + if fp, ok := w.wants[wk]; ok && fp == have.FP { + w.moved(delivery.AssignerWantsBucket, wk) + } +} + +func (w *Watchman) have(node, module string) string { + key, err := delivery.HaveKey(node, module) + if err != nil { + return "" + } + return w.haves[key] +} + +func shortFP(fp string) string { + hex := strings.TrimPrefix(fp, delivery.SourcePrefix2) + if len(hex) > 12 { + hex = hex[:12] + } + return hex +} diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index a5c8bee..e81613b 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -57,6 +57,8 @@ type Watchman struct { presence map[string]presence pulses map[string]*pulse // live pulses, by pulse key stuck map[string]delivery.Stuck + wants map[string]string // the fingerprint of each want that stands, by want key + haves map[string]string // the fingerprint each node runs of a module, by have key bound time.Duration // stepBound is how long a build step may run once a node took the ask; a test shortens it. stepBound time.Duration @@ -65,7 +67,7 @@ type Watchman struct { // New is a watchman writing to out, reading the time from now. func New(out Out, now func() time.Time) *Watchman { return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, - pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, bound: delivery.PulseMoveBound, stepBound: StepBound} + pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index eef647d..28ea6f0 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "slices" + "strings" "testing" "time" @@ -531,3 +532,91 @@ func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908 t.Errorf("stuck %v pulses %v", o2.stuck, o2.pulses) } } + +// 1906/D6 (ADR 0539 §19): a want is a pulse owed by the node's agent from when the assigner wrote it, until the have names +// its fingerprint. +func wantValue(t *testing.T, fp string, removed bool) []byte { + t.Helper() + w := delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed} + if removed { + w.Composed = "" + } + b, err := delivery.EncodeWant(w) + if err != nil { + t.Fatal(err) + } + return b +} + +func haveValue(t *testing.T, fp string) []byte { + t.Helper() + b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + if err != nil { + t.Fatal(err) + } + return b +} + +var ( + fpOld = delivery.SourcePrefix2 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + fpNew = delivery.SourcePrefix2 + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +) + +func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) { + w, o, c := setup() + c.t = t0.Add(10 * time.Second) + w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0) + pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web") + p, ok := o.pulses[pk] + if !ok || p.Key != "want.ace.web" || !strings.Contains(p.Owes, "ace") || !strings.Contains(p.Owes, "web") { + t.Fatalf("pulses = %v", o.pulses) + } + w.Have("ace.web", haveValue(t, fpOld), Put) // the node still runs the old one + if _, ok := o.pulses[pk]; !ok { + t.Fatalf("a have of another fingerprint ended the pulse") + } + w.Have("ace.web", haveValue(t, fpNew), Put) + if _, ok := o.pulses[pk]; ok { + t.Errorf("the have naming the want's fingerprint left the pulse standing: %v", o.pulses) + } + if len(o.stuck) != 0 { + t.Errorf("stuck = %v", o.stuck) + } +} + +func TestAHaveThatCameBeforeTheWantEndsItAtOnceAndARemovedWantHasNoPulse(t *testing.T) { + w, o, c := setup() + c.t = t0.Add(time.Second) + w.Have("ace.web", haveValue(t, fpNew), Put) + w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0) + w.Want("want.ace.mail", wantValue(t, "", true), Put, t0) + if len(o.pulses) != 0 { + t.Errorf("pulses = %v", o.pulses) + } + // the want moving on to a new fingerprint makes a new pulse; the key `want.ace.node` and a held key are left + w.Want("want.ace.web", wantValue(t, fpOld, false), Put, t0.Add(time.Second)) + w.Want("want.ace.node", []byte(`{}`), Put, t0) + w.Want("held.web.bbbb", []byte(`{}`), Put, t0) + if len(o.pulses) != 1 { + t.Errorf("pulses = %v, want the web want's only", o.pulses) + } +} + +func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) { + o, c := newOut(), &clock{t: t0} + w := New(o, c.now) + w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0) + pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web") + c.t = t0.Add(31 * time.Minute) + w.Pulse(pk, Expired, c.t) + if s, ok := o.stuck[stuckKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")]; !ok || s.Moved || !s.Since.Equal(t0) { + t.Fatalf("stuck = %v", o.stuck) + } + w2, o2, c2 := setup() + c2.t = t0.Add(time.Second) + w2.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0) + w2.Want("want.ace.web", nil, Delete, t0) + if len(o2.pulses) != 0 { + t.Errorf("a deleted want left its pulse: %v", o2.pulses) + } +} diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index d0344b9..58e2171 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -81,7 +81,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 } reads := slices.Clone(m.Reads) slices.Sort(reads) - if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps"}) { + if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-delivery-assigner.wants", "node-engine.have"}) { t.Errorf("reads = %v", reads) } for _, k := range []string{"emits", "context", "items", "consumes", "provides", "claims"} { diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 8c4c713..c1c128f 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -5,7 +5,9 @@ "reads": [ "mesh-build-go.steps", "mesh-build-image.steps", - "mesh-build-node.steps" + "mesh-build-node.steps", + "mesh-delivery-assigner.wants", + "node-engine.have" ], "state": [ { "name": "presence", "history": 20 },