Compare commits
31
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3e863884fd | ||
|
|
c21bed20a3 | ||
|
|
aa31e85665 | ||
|
|
36a1fd197f | ||
|
|
9a5b3d4881 | ||
|
|
e263049551 | ||
|
|
0f7bc9f531 | ||
|
|
da31740f00 | ||
|
|
a3815051cc | ||
|
|
8280506cee | ||
|
|
3b00f3890d | ||
|
|
fcac03a4f2 | ||
|
|
14c20b21ee | ||
|
|
333e0cd07d | ||
|
|
03e5951dc5 | ||
|
|
37d0ebba7a | ||
|
|
86fd0bbe96 | ||
|
|
37a5ab8f1c | ||
|
|
55f19283f4 | ||
|
|
e7a3ff1e7e | ||
|
|
f947070b7b | ||
|
|
04c2c60f02 | ||
|
|
0df20b5d77 | ||
|
|
806c8bf0e4 | ||
|
|
12b01badb0 | ||
|
|
7b18696cd5 | ||
|
|
204a59dab4 | ||
|
|
f048e2b443 | ||
|
|
9d2e9dc6c9 | ||
|
|
024a3787da | ||
|
|
4fd6b934c8 |
@@ -5,6 +5,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -17,9 +18,12 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events
|
||||
@@ -73,6 +77,10 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The operator-channel seat's stream of conditions, which the assigner says its conditions on.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CONDITIONS", Subjects: []string{"mesh.seat.operator-channel.accept.condition.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The Go build seat's work queue, as the broker derives it from the seat's accepted kinds (delivery.AskStream).
|
||||
seat, _ := delivery.AskStream(delivery.AskGo)
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: seat, Subjects: []string{"mesh.seat.node-build-go.accept.>"},
|
||||
@@ -85,6 +93,14 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
// start runs the program as the node does, through run with its environment and a broker file; it answers the
|
||||
// channel run's error arrives on.
|
||||
func (b *bed) start(t *testing.T) <-chan error {
|
||||
t.Helper()
|
||||
done, _ := b.startStoppable(t)
|
||||
return done
|
||||
}
|
||||
|
||||
// startStoppable is start, and answers the function that stops this one program (a replaced assigner is stopped, then
|
||||
// another started).
|
||||
func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(b.dir, "broker")
|
||||
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"})
|
||||
@@ -94,15 +110,19 @@ func (b *bed) start(t *testing.T) <-chan error {
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
ctx, stop := context.WithCancel(b.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(ctx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() {
|
||||
go func() {
|
||||
done <- run(ctx, func(k string) string { return env[k] }, io.Discard)
|
||||
close(done) // a program that ended on its own (a refused start) is not waited for again at the cleanup
|
||||
}()
|
||||
stopped := func() {
|
||||
stop()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
}
|
||||
})
|
||||
return done
|
||||
}
|
||||
t.Cleanup(stopped)
|
||||
return done, stopped
|
||||
}
|
||||
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
|
||||
@@ -159,6 +179,27 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
|
||||
// move to it, in mesh-controller_composed.
|
||||
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
|
||||
t.Helper()
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat(string(d), 64)
|
||||
cd := delivery.ComposedDigest{Digest: digest, Judgement: judgement}
|
||||
if judgement != delivery.JudgedNeither {
|
||||
cd.Against, cd.Words = "sha256:"+strings.Repeat("0", 64), "cannot be told"
|
||||
}
|
||||
v, err := delivery.EncodeComposedDigest(cd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
return digest
|
||||
}
|
||||
|
||||
// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest.
|
||||
func (b *bed) manifest(t *testing.T, module, body string) string {
|
||||
t.Helper()
|
||||
@@ -240,10 +281,10 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1297/B4, B5 on the bus: the controller sent F to g14 while the canary ace does not run F healthy yet. Started whole,
|
||||
// the assigner wants F on the canary, writes the difference on g14 explained by rule 3, and on ace one unexplained (the
|
||||
// controller sends ace the older fingerprint while ace wants F).
|
||||
func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||
// 1297/B4 on the bus, 1906/D5: the controller sends ace and g14 builds while the canary ace does not run F healthy yet.
|
||||
// Started whole, the assigner wants F on the canary, holds g14 back with no want, and writes no difference at all: there
|
||||
// is no send to compare with (ADR 0539 §8).
|
||||
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
@@ -258,6 +299,8 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.compose(t, "g14", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||
@@ -266,8 +309,6 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||
}
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
|
||||
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-8"), []byte(`{"id":"b-8","repository":"novox/web","source-fingerprint":"`+fOld+`"}`))
|
||||
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-9"), []byte(`{"id":"b-9","repository":"novox/web","source-fingerprint":"`+fNew+`"}`))
|
||||
b.start(t)
|
||||
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
@@ -275,15 +316,9 @@ func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||
if err != nil || want.FP != fNew {
|
||||
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||
}
|
||||
dkey, _ := delivery.DifferenceKey("g14", "web")
|
||||
d, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, dkey))
|
||||
if err != nil || d.Rule != delivery.RuleCanary || d.Sent != fNew {
|
||||
t.Fatalf("g14's difference = %+v, %v; want rule 3", d, err)
|
||||
}
|
||||
akey, _ := delivery.DifferenceKey("ace", "web")
|
||||
a, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, akey))
|
||||
if err != nil || !a.Unexplained || a.Want != fNew || a.Sent != fOld {
|
||||
t.Fatalf("ace's difference = %+v, %v; want it unexplained", a, err)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
|
||||
t.Errorf("differences were written: %v", keys)
|
||||
}
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
|
||||
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
|
||||
@@ -315,3 +350,426 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
// conditionsSaid reads what the assigner said on the operator channel's subject for conditions, by condition key.
|
||||
func (b *bed) conditionsSaid(t *testing.T) map[string]asks.Condition {
|
||||
t.Helper()
|
||||
stream, err := b.js.Stream(b.ctx, "CONDITIONS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := map[string]asks.Condition{}
|
||||
cons, err := stream.CreateOrUpdateConsumer(b.ctx, jetstream.ConsumerConfig{AckPolicy: jetstream.AckNonePolicy})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
batch, err := cons.FetchNoWait(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
c, err := asks.DecodeCondition(m.Data())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out[c.Key] = c
|
||||
}
|
||||
if n == 0 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19), whole on the bus: a module holding data whose new fingerprint failed its health check on the
|
||||
// canary is held, not set back, and asked about (restore it, or leave it stopped); the condition names the module, the node
|
||||
// and the check's words.
|
||||
func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("d", 64)
|
||||
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, "mail", mod)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.mail", failed)
|
||||
wantKey, _ := delivery.WantKey("ace", "mail")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
b.putJSON(t, delivery.AssignerWantsBucket, wantKey, want)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.mail", []byte(`{"node":"ace","module":"mail","build":"b-1"}`))
|
||||
b.start(t)
|
||||
|
||||
akey, _ := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || len(a.Ask.Options) != 2 || a.Ask.Options[0].ID != "restore" || a.Ask.Options[1].ID != "leave-stopped" {
|
||||
t.Fatalf("the restore ask = %+v, %v", a, err)
|
||||
}
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wantKey)); err != nil || w.FP != fNew {
|
||||
t.Errorf("the data module was set back: want = %+v, %v", w, err)
|
||||
}
|
||||
key := ""
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.State == asks.StateRaised && c.Subject.ID == "mail" {
|
||||
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
|
||||
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
|
||||
}
|
||||
key = k
|
||||
}
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
t.Fatal("no condition for the held fingerprint was said")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D8 (scenario D of acceptance test 18, ADR 0539 §19), whole on the bus: the assigner is stopped while it writes the wants
|
||||
// of forty modules and another is started; every module's want is written, and each exactly once, by revision: nothing lost
|
||||
// and nothing written again.
|
||||
func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
const modules = 40
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
names := make([]string, modules)
|
||||
for i := range names {
|
||||
name := fmt.Sprintf("mod%02d", i)
|
||||
names[i] = name
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat(string(rune('a'+i%6)), 63) + fmt.Sprintf("%x", i%16)
|
||||
digest := b.manifest(t, name, fmt.Sprintf(`{"module":%q,"version":"1"}`, name))
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, name, mod)
|
||||
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", name, fp, 'c', delivery.JudgedNeither)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(fmt.Sprintf(`{"node":"ace","module":%q,"build":"b-%d"}`, name, i)))
|
||||
}
|
||||
count := func() int {
|
||||
keys, err := b.buckets[delivery.AssignerWantsBucket].Keys(b.ctx)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return len(keys)
|
||||
}
|
||||
_, stop := b.startStoppable(t)
|
||||
for deadline := time.Now().Add(20 * time.Second); count() == 0 && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
||||
}
|
||||
stop() // replaced, wherever its pass had got to
|
||||
b.start(t)
|
||||
for deadline := time.Now().Add(30 * time.Second); count() < modules && time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
}
|
||||
time.Sleep(time.Second) // and no pass of the second writes any again
|
||||
for _, name := range names {
|
||||
key, _ := delivery.WantKey("ace", name)
|
||||
history, err := b.buckets[delivery.AssignerWantsBucket].History(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Errorf("the want of %s was never written: %v", name, err)
|
||||
continue
|
||||
}
|
||||
if len(history) != 1 {
|
||||
t.Errorf("the want of %s was written %d times, want once", name, len(history))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §6), whole on the bus: a want waits for the composition of its module for the node, names its digest once it
|
||||
// appears, and a new revision of the composition at the same fingerprint is written as a new want; the node's own want
|
||||
// `want.<node>.node` is written once the node's composition exists, and again when its digest changes.
|
||||
func TestAWantWaitsForItsComposition(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("e", 64)
|
||||
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
time.Sleep(time.Second)
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
t.Fatalf("a want was written before the composition existed: %s", e.Value())
|
||||
}
|
||||
first := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.FP != fNew || w.Composed != first {
|
||||
t.Fatalf("the want = %+v, %v; want F naming %s", w, err, first)
|
||||
}
|
||||
second := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, wkey); err == nil {
|
||||
if w, err := delivery.DecodeWant(e.Value()); err == nil && w.Composed == second {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if w, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)); err != nil || w.Composed != second {
|
||||
t.Fatalf("after a new revision at the same fingerprint the want = %+v, %v; want it naming %s", w, err, second)
|
||||
}
|
||||
// the node's own want
|
||||
nkey, _ := delivery.WantNodeKey("ace")
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
|
||||
t.Fatalf("the node's own want was written before its composition: %s", e.Value())
|
||||
}
|
||||
ownKey, _ := delivery.ComposedNodeKey("ace")
|
||||
own, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither})
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, ownKey, own)
|
||||
if w, err := delivery.DecodeNodeWant(b.get(t, delivery.AssignerWantsBucket, nkey)); err != nil || w.Composed != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the node's own want = %+v, %v", w, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fixture is a module web (or the manifest given) at fp on the node ace, which is online: built, granted and assigned.
|
||||
func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
|
||||
t.Helper()
|
||||
digest := b.manifest(t, name, manifest)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/" + name, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, name, mod)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||
bkey, _ := delivery.BuilderBuildKey(name, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.ControllerGrantKey(name, fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace."+name, []byte(`{"node":"ace","module":"`+name+`","build":"b-1"}`))
|
||||
}
|
||||
|
||||
func (b *bed) putWant(t *testing.T, node, module string, w delivery.Want) {
|
||||
t.Helper()
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
v, err := delivery.EncodeWant(w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.AssignerWantsBucket, key, v)
|
||||
}
|
||||
|
||||
// wantNow waits until the want of module on node satisfies ok and answers it.
|
||||
func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.Want) bool) delivery.Want {
|
||||
t.Helper()
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
var last delivery.Want
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, key); err == nil {
|
||||
if w, err := delivery.DecodeWant(e.Value()); err == nil {
|
||||
last = w
|
||||
if ok(w) {
|
||||
return w
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("never saw: %s; the want is %+v", what, last)
|
||||
return last
|
||||
}
|
||||
|
||||
// answer says the teller wrote the operator's answer to the ask at key.
|
||||
func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") }
|
||||
|
||||
// answerTo is answer to the ask with this id (the router's warrant names it).
|
||||
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: askID + "@2026-10-24T12:00:00Z", Words: "the operator"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.AnswersBucket, key, v)
|
||||
}
|
||||
|
||||
func (b *bed) hasKey(bucket, key string) bool {
|
||||
_, err := b.buckets[bucket].Get(b.ctx, key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19), whole on the bus: the operator's release of a held fingerprint writes its want again, by the rules,
|
||||
// and clears the condition.
|
||||
func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.compose(t, "ace", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy) // it ran the older fingerprint healthy before
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
b.start(t)
|
||||
w := b.wantNow(t, "ace", "web", "set back to the older fingerprint", func(w delivery.Want) bool { return w.FP == fOld })
|
||||
if w.Newest != fNew {
|
||||
t.Fatalf("the set-back want = %+v", w)
|
||||
}
|
||||
hold, _ := delivery.HoldAskKey("web", fNew)
|
||||
b.get(t, delivery.AssignerAsksBucket, hold)
|
||||
b.answer(t, hold, "release")
|
||||
w = b.wantNow(t, "ace", "web", "the held fingerprint's want written again", func(w delivery.Want) bool { return w.FP == fNew })
|
||||
if w.Composed != "sha256:"+strings.Repeat("b", 64) || w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
cleared := false
|
||||
for _, c := range b.conditionsSaid(t) {
|
||||
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared)
|
||||
}
|
||||
if cleared {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("the condition of the held fingerprint was not cleared")
|
||||
}
|
||||
|
||||
// askNow waits for the backup ask at key to be the one numbered asking and answers it.
|
||||
func (b *bed) askNow(t *testing.T, key string, asking int) delivery.AssignerAsk {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.AssignerAsksBucket].Get(b.ctx, key); err == nil {
|
||||
if a, err := delivery.DecodeAssignerAsk(key, e.Value()); err == nil && a.Asking == asking {
|
||||
return a
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the ask at %s never became asking %d", key, asking)
|
||||
return delivery.AssignerAsk{}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §16), whole on the bus: a module held on a node because its backup failed is asked about, and the answer
|
||||
// is written into the node's want; an answer "again" followed by another failure is asked again, and its answer is another want
|
||||
// value, so the node retries; an old answer is never taken for a new revision at the same fingerprint.
|
||||
func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
first := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: first})
|
||||
held, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", held)
|
||||
b.start(t)
|
||||
akey, _ := delivery.BackupAskKey("ace", "web", fNew)
|
||||
a := b.askNow(t, akey, 0)
|
||||
if !strings.Contains(a.Ask.Explanation, "no space left") || !strings.Contains(a.Ask.Explanation, first) {
|
||||
t.Fatalf("the backup ask = %+v", a)
|
||||
}
|
||||
// an answer to another ask is no answer to this one
|
||||
b.answerTo(t, akey, delivery.BackupSkip, "someone-else")
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
|
||||
t.Fatalf("an answer to another ask was taken: %+v", w)
|
||||
}
|
||||
b.answerTo(t, akey, delivery.BackupAgain, a.Ask.ID)
|
||||
w1 := b.wantNow(t, "ace", "web", "the answer in the want", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain })
|
||||
if w1.FP != fNew || w1.Composed != first {
|
||||
t.Errorf("the want = %+v", w1)
|
||||
}
|
||||
// the node retries and the backup fails again: a new have value, asked again under the next number
|
||||
again, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left, again"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", again)
|
||||
a2 := b.askNow(t, akey, 2)
|
||||
if a2.Ask.ID == a.Ask.ID {
|
||||
t.Fatalf("asked again under the same id")
|
||||
}
|
||||
b.answerTo(t, akey, delivery.BackupAgain, a2.Ask.ID)
|
||||
w2 := b.wantNow(t, "ace", "web", "the second answer another want value", func(w delivery.Want) bool { return w.Backup == delivery.BackupAgain && w.Words != w1.Words })
|
||||
if w2.Words == w1.Words {
|
||||
t.Errorf("the second answer left the want as it was: %+v", w2)
|
||||
}
|
||||
// a new revision at the same fingerprint, whose backup fails as well, is asked, and the old answer is not taken for it
|
||||
second := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: second})
|
||||
a3 := b.askNow(t, akey, 3)
|
||||
if !strings.Contains(a3.Ask.Explanation, second) {
|
||||
t.Errorf("the ask for the new revision = %+v", a3)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Backup != "" {
|
||||
t.Errorf("the old answer was taken for the new revision: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
const recorded = `{"module":"web","version":"1","upgrade":{"policy":"record","why":"a person takes each build"}}`
|
||||
|
||||
// 1906/D4 (ADR 0539 §14), whole on the bus: a move of a module of policy record waits for its ask's yes when judged to recreate
|
||||
// a container, to move data or not to be judgeable, at a new fingerprint and at a new revision of the same alike; one judged
|
||||
// neither is written at once.
|
||||
func TestARecordMoveWaitsForTheYesAndANeitherMoveDoesNot(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "web", fNew, recorded)
|
||||
old := "sha256:" + strings.Repeat("a", 64)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fOld, Composed: old})
|
||||
nw := b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedRecreates)
|
||||
b.start(t)
|
||||
akey, _ := delivery.RecordAskKey("ace", "web", nw)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || !strings.Contains(a.Ask.Explanation, "recreates a container") {
|
||||
t.Fatalf("the record ask = %+v, %v", a, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the old want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the move was written before the yes: %+v", w)
|
||||
}
|
||||
b.answer(t, akey, assign.OptionYes)
|
||||
b.wantNow(t, "ace", "web", "the move after the yes", func(w delivery.Want) bool { return w.FP == fNew && w.Composed == nw })
|
||||
// a new revision at the same fingerprint, judged to move data, waits as well
|
||||
rev := b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedMovesData)
|
||||
rkey, _ := delivery.RecordAskKey("ace", "web", rev)
|
||||
b.get(t, delivery.AssignerAsksBucket, rkey)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "web", "the want", func(delivery.Want) bool { return true }); w.Composed != nw {
|
||||
t.Fatalf("the revision was written before the yes: %+v", w)
|
||||
}
|
||||
// and a revision judged neither is written at once
|
||||
free := b.compose(t, "ace", "web", fNew, 'd', delivery.JudgedNeither)
|
||||
b.wantNow(t, "ace", "web", "a move judged neither written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15), whole on the bus: the bus's new fingerprint is not written, and a revision of its composition at the
|
||||
// fingerprint it runs that differs only in its user list is written at once, with the shape the controller gives it: judged
|
||||
// recreates (the file is mounted), policy record from the bus, user-list-only; no record ask is made.
|
||||
func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
b.fixture(t, "bus", fNew, `{"module":"bus","version":"1","provides":[{"name":"mesh-bus"}]}`)
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.bus", running)
|
||||
b.putWant(t, "ace", "bus", delivery.Want{FP: fOld, Composed: "sha256:" + strings.Repeat("a", 64)})
|
||||
b.compose(t, "ace", "bus", fNew, 'b', delivery.JudgedNeither)
|
||||
b.start(t)
|
||||
time.Sleep(time.Second)
|
||||
if w := b.wantNow(t, "ace", "bus", "the want", func(delivery.Want) bool { return true }); w.FP != fOld {
|
||||
t.Fatalf("the assigner moved the bus to a new fingerprint: %+v", w)
|
||||
}
|
||||
// the controller composes the bus at the fingerprint it runs with a new user list
|
||||
key, _ := delivery.ComposedKey("ace", "bus", fOld)
|
||||
free := "sha256:" + strings.Repeat("e", 64)
|
||||
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: free, Judgement: delivery.JudgedRecreates, Against: "sha256:" + strings.Repeat("a", 64),
|
||||
Words: "the user list file is mounted", UserListOnly: true, Policy: delivery.PolicyRecord, PolicyFrom: "the bus"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.ControllerComposedBucket, key, v)
|
||||
w := b.wantNow(t, "ace", "bus", "the user-list revision written at once", func(w delivery.Want) bool { return w.Composed == free })
|
||||
if w.FP != fOld {
|
||||
t.Errorf("the want = %+v, want it at the fingerprint the bus runs", w)
|
||||
}
|
||||
rkey, _ := delivery.RecordAskKey("ace", "bus", free)
|
||||
if b.hasKey(delivery.AssignerAsksBucket, rkey) {
|
||||
t.Errorf("the user-list revision was asked")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -42,6 +43,8 @@ var readBuckets = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.BuildsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ManifestsBucket,
|
||||
@@ -130,7 +133,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, logger)
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
|
||||
}
|
||||
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
@@ -138,6 +141,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
var watched = []string{
|
||||
provider.AssignmentsBucket,
|
||||
delivery.ControllerGrantsBucket,
|
||||
delivery.ControllerComposedBucket,
|
||||
delivery.AnswersBucket,
|
||||
delivery.BuilderBuildsBucket,
|
||||
delivery.CandidatesBucket,
|
||||
delivery.ModulesBucket,
|
||||
@@ -151,7 +156,7 @@ var watched = []string{
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -215,7 +220,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *lo
|
||||
case <-settled:
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
wake()
|
||||
for {
|
||||
@@ -230,3 +235,20 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, logger *lo
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||
// (ADR 0437 clause 9).
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject(delivery.AssignerModule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := asks.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, data)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||
@@ -19,76 +21,43 @@ type passer struct {
|
||||
w *world
|
||||
logf func(string, ...any)
|
||||
sdk string
|
||||
asks *composeAsks
|
||||
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(asks.Condition) error
|
||||
said map[string]bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
asks *composeAsks
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.compositions(ctx)
|
||||
rules := p.wants(ctx)
|
||||
p.differences(ctx, rules)
|
||||
p.wants(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
}
|
||||
|
||||
// differences writes, for each node and module the assigner wants or the controller sends, the difference between the
|
||||
// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that
|
||||
// ended; its history keeps what it was.
|
||||
func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) {
|
||||
_, assignments := p.w.assigned()
|
||||
wanted := p.w.wantsOf()
|
||||
trunk := p.w.trunk()
|
||||
pairs := map[[2]string]bool{}
|
||||
for _, a := range assignments {
|
||||
pairs[[2]string{a.Node, a.Module}] = true
|
||||
}
|
||||
for module, byNode := range wanted {
|
||||
for node := range byNode {
|
||||
pairs[[2]string{node, module}] = true
|
||||
}
|
||||
}
|
||||
written := p.w.read(delivery.AssignerDifferencesBucket)
|
||||
keep := map[string]bool{}
|
||||
for pair := range pairs {
|
||||
node, module := pair[0], pair[1]
|
||||
key, err := delivery.DifferenceKey(node, module)
|
||||
// nodeWants writes each node's own want, `want.<node>.node`, naming the digest of the node's own composition, once the
|
||||
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
|
||||
func (p *passer) nodeWants(ctx context.Context) {
|
||||
written := p.w.read(delivery.AssignerWantsBucket)
|
||||
for node, d := range p.w.nodeComposed() {
|
||||
key, err := delivery.WantNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var want *delivery.Want
|
||||
if w, ok := wanted[module][node]; ok {
|
||||
want = &w
|
||||
}
|
||||
var sent assign.Sent
|
||||
if a, ok := assignments[node+"."+module]; ok {
|
||||
sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)}
|
||||
}
|
||||
d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node])
|
||||
if !ok {
|
||||
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Composed == d.Digest {
|
||||
continue
|
||||
}
|
||||
keep[key] = true
|
||||
if before, err := delivery.DecodeDifference(written[key]); err == nil {
|
||||
before.V = 0
|
||||
if delivery.SameDifference(before, d) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) })
|
||||
}
|
||||
for key := range written {
|
||||
if keep[key] {
|
||||
continue
|
||||
}
|
||||
if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil {
|
||||
p.logf("the difference %s that ended could not be deleted: %v", key, err)
|
||||
continue
|
||||
}
|
||||
p.w.put(delivery.AssignerDifferencesBucket, key, nil)
|
||||
want := delivery.NodeWant{Composed: d.Digest, Words: "the node's own composition"}
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
|
||||
}
|
||||
}
|
||||
|
||||
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
|
||||
// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node
|
||||
// back, for the comparison.
|
||||
func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding {
|
||||
// the hold of a fingerprint whose health check failed. The assigner compares its wants with nothing the controller
|
||||
// sends: the differences bucket is kept read-only with its history (ADR 0539 §8).
|
||||
func (p *passer) wants(ctx context.Context) {
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
@@ -109,7 +78,6 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
rules := map[string]map[string]assign.Holding{}
|
||||
names := map[string]bool{}
|
||||
for name := range trunk {
|
||||
names[name] = true
|
||||
@@ -119,18 +87,32 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
}
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]}
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name],
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
return err == nil && p.answeredWith(key, assign.OptionRestore)
|
||||
},
|
||||
Allowed: func(node, digest string) bool {
|
||||
key, err := delivery.RecordAskKey(node, name, digest)
|
||||
return err == nil && p.answeredWith(key, assign.OptionYes)
|
||||
}}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if m.Released {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
}
|
||||
last := map[string]string{}
|
||||
for _, node := range m.Nodes {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
rules[name] = out.Rules
|
||||
if out.Held != nil {
|
||||
key, err := delivery.HeldKey(name, fp)
|
||||
if err == nil {
|
||||
@@ -140,6 +122,13 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
}
|
||||
if out.Held != nil || m.Held {
|
||||
p.holdAsk(ctx, name, fp)
|
||||
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
||||
}
|
||||
for _, node := range out.Restore {
|
||||
p.restoreAsk(ctx, name, node, fp)
|
||||
}
|
||||
for _, need := range out.Record {
|
||||
p.recordAsk(ctx, name, fp, need)
|
||||
}
|
||||
for node, want := range out.Wants {
|
||||
key, err := delivery.WantKey(node, name)
|
||||
@@ -151,7 +140,6 @@ func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
return rules
|
||||
}
|
||||
|
||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||
@@ -230,3 +218,152 @@ func (p *passer) holdAsk(ctx context.Context, module, fp string) {
|
||||
a := assign.HoldAsk(key, module, fp, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
||||
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
||||
if fresh != nil {
|
||||
return *fresh
|
||||
}
|
||||
key, err := delivery.HeldKey(module, fp)
|
||||
if err != nil {
|
||||
return delivery.Held{}
|
||||
}
|
||||
h, _ := delivery.DecodeHeld(p.w.read(delivery.AssignerWantsBucket)[key])
|
||||
return h
|
||||
}
|
||||
|
||||
// heldCondition raises, once, the condition for a fingerprint the assigner holds (ADR 0429 §6, ADR 0539 §19): the module, the
|
||||
// node whose health check failed and the check's words. A module holding data needs the operator's answer to its restore ask.
|
||||
func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *delivery.Held, data bool) {
|
||||
h := p.heldHold(module, fp, fresh)
|
||||
c := assign.HeldCondition(module, fp, h.Node, h.Words, true, data)
|
||||
if p.say == nil || p.said[c.Key+".raised"] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the held %s could not be said, and is said at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
}
|
||||
|
||||
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
||||
func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
|
||||
key, err := delivery.RestoreAskKey(node, module, fp)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
words := p.heldHold(module, fp, nil).Words
|
||||
a := assign.RestoreAsk(key, node, module, fp, words, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the
|
||||
// operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19).
|
||||
func (p *passer) answeredWith(key, option string) bool {
|
||||
a, ok := p.w.answer(key)
|
||||
return ok && a.Option == option
|
||||
}
|
||||
|
||||
// clearHeld clears, once, the condition of a hold the operator released.
|
||||
func (p *passer) clearHeld(module, fp string) {
|
||||
c := assign.HeldCondition(module, fp, "", "", false, false)
|
||||
if p.say == nil || p.said[c.Key+".cleared"] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key+".cleared"] = true
|
||||
}
|
||||
|
||||
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
|
||||
func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.RecordNeed) {
|
||||
key, err := delivery.RecordAskKey(need.Node, module, need.Digest)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
a := assign.RecordAsk(key, need.Node, module, fp, need, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// backups sees to the modules held on a node because the backup before an apply failed (the have's phase held, ADR 0539 §16):
|
||||
// it asks whether to back up again or go on without a backup, raises the condition naming node, module and the backup's words,
|
||||
// and writes the answer into the node's want for the module, which the node agent reads from it and from nothing else.
|
||||
//
|
||||
// The ask is bound to the composition (it names the digest, and a new revision at the same fingerprint is asked again under a
|
||||
// new number) and an answer counts only if the router's warrant names the id of the ask now at the key (the warrant's first part,
|
||||
// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again
|
||||
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
|
||||
// in a new have value after the answer was written.
|
||||
func (p *passer) backups(ctx context.Context) {
|
||||
wanted := p.w.wantsOf()
|
||||
for key, value := range p.w.read(delivery.HaveBucket) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
have, err := delivery.DecodeHave(value)
|
||||
if err != nil || have.Phase != delivery.PhaseHeld {
|
||||
continue
|
||||
}
|
||||
want, ok := wanted[module][node]
|
||||
if !ok || want.Removed {
|
||||
continue
|
||||
}
|
||||
akey, err := delivery.BackupAskKey(node, module, want.FP)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
current, there := delivery.DecodeAssignerAsk(akey, p.w.read(delivery.AssignerAsksBucket)[akey])
|
||||
if there != nil || assign.BackupAskDigest(current) != want.Composed {
|
||||
after := -1
|
||||
if there == nil {
|
||||
after = current.Asking
|
||||
}
|
||||
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(after))
|
||||
continue
|
||||
}
|
||||
answer, answered := p.w.answer(akey)
|
||||
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != current.Ask.ID ||
|
||||
(answer.Option != delivery.BackupAgain && answer.Option != delivery.BackupSkip) {
|
||||
continue // not answered yet: the ask stands
|
||||
}
|
||||
words := assign.BackupAnswerWords(answer.Option, current.Asking)
|
||||
haveChanges := p.w.changed(delivery.HaveBucket, key)
|
||||
if want.Backup != answer.Option || want.Words != words {
|
||||
want.Backup, want.Words = answer.Option, words
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
w := want
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(w) })
|
||||
p.answeredAt[current.Ask.ID] = haveChanges
|
||||
}
|
||||
continue
|
||||
}
|
||||
if answer.Option != delivery.BackupAgain {
|
||||
continue
|
||||
}
|
||||
// The answer is in the want. The node retried and the backup failed again when the have took another value since.
|
||||
if seen, known := p.answeredAt[current.Ask.ID]; !known {
|
||||
p.answeredAt[current.Ask.ID] = haveChanges // after a restart: wait for the next failure
|
||||
} else if haveChanges > seen {
|
||||
p.backupAsk(ctx, akey, node, module, want, have.Detail, assign.NextBackupAsking(current.Asking))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// backupAsk writes the backup ask at its asking and raises the condition of the failed backup, once.
|
||||
func (p *passer) backupAsk(ctx context.Context, key, node, module string, want delivery.Want, words string, asking int) {
|
||||
a := assign.BackupAsk(key, node, module, want.FP, want.Composed, words, asking, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
c := assign.BackupCondition(node, module, want.FP, words, true)
|
||||
if p.say == nil || p.said[c.Key] {
|
||||
return
|
||||
}
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of the failed backup of %s on %s could not be said: %v", module, node, err)
|
||||
return
|
||||
}
|
||||
p.said[c.Key] = true
|
||||
}
|
||||
|
||||
@@ -44,8 +44,8 @@ type world struct {
|
||||
manifests map[string]parsed
|
||||
// healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to.
|
||||
healthy map[string][]string
|
||||
// sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one.
|
||||
sent map[string]string
|
||||
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
|
||||
changes map[string]uint64
|
||||
}
|
||||
|
||||
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||
@@ -56,7 +56,7 @@ type parsed struct {
|
||||
}
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}}
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
|
||||
}
|
||||
|
||||
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||
@@ -71,6 +71,7 @@ func (w *world) put(bucket, key string, value []byte) {
|
||||
return
|
||||
}
|
||||
w.values[bucket][key] = value
|
||||
w.changes[bucket+"|"+key]++
|
||||
if bucket == delivery.HaveBucket {
|
||||
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||
@@ -349,30 +350,55 @@ func (w *world) built(module, fp string) bool {
|
||||
return err == nil && b.State == delivery.BuildDone
|
||||
}
|
||||
|
||||
// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record
|
||||
// (mesh-controller_builds `<module>.build.<id>`) once and kept; empty when the record names none.
|
||||
func (w *world) sentFP(ctx context.Context, get getter, module, build string) string {
|
||||
if build == "" {
|
||||
return ""
|
||||
}
|
||||
w.mu.Lock()
|
||||
fp, ok := w.sent[build]
|
||||
w.mu.Unlock()
|
||||
if ok {
|
||||
return fp
|
||||
}
|
||||
value, err := get(ctx, delivery.BuildKey(module, build))
|
||||
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
|
||||
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
|
||||
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
|
||||
key, err := delivery.ComposedKey(node, module, fp)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var rec delivery.BuildRecord
|
||||
if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) {
|
||||
return ""
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
w.mu.Lock()
|
||||
w.sent[build] = rec.SourceFingerprint
|
||||
value, ok := w.values[delivery.ControllerComposedBucket][key]
|
||||
w.mu.Unlock()
|
||||
return rec.SourceFingerprint
|
||||
if !ok {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(value)
|
||||
return d, err == nil
|
||||
}
|
||||
|
||||
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
|
||||
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
|
||||
out := map[string]delivery.ComposedDigest{}
|
||||
for key, value := range w.read(delivery.ControllerComposedBucket) {
|
||||
node, _, _, isNode, err := delivery.ParseComposedKey(key)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if d, err := delivery.DecodeComposedDigest(value); err == nil {
|
||||
out[node] = d
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// answer is the teller's answer to the ask at key (mesh-delivery-teller_answers, the same key), and whether there is one.
|
||||
func (w *world) answer(key string) (delivery.Answer, bool) {
|
||||
w.mu.Lock()
|
||||
value, ok := w.values[delivery.AnswersBucket][key]
|
||||
w.mu.Unlock()
|
||||
if !ok {
|
||||
return delivery.Answer{}, false
|
||||
}
|
||||
a, err := delivery.DecodeAnswer(value)
|
||||
return a, err == nil
|
||||
}
|
||||
|
||||
// changed is how many values the key has taken in this process (see world.changes).
|
||||
func (w *world) changed(bucket, key string) uint64 {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.changes[bucket+"|"+key]
|
||||
}
|
||||
|
||||
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
|
||||
|
||||
@@ -111,3 +111,17 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
|
||||
t.Errorf("xterm: waits %q, unreadable %q, want read", waits, ch.Unreadable["xterm"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
|
||||
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
|
||||
w := newWorld()
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
|
||||
w.put(delivery.HaveBucket, "ace.mail", []byte(`{"v":1}`))
|
||||
if got := w.changed(delivery.HaveBucket, "ace.web"); got != 2 {
|
||||
t.Errorf("changed = %d, want 2", got)
|
||||
}
|
||||
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
|
||||
t.Errorf("changed = %d for a key never seen", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980 h1:ZEEeYibEPD1rngOAqbODzRY5rUStGC46OpyQwtNCYbs=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024125710-bfc3883af980/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -2,6 +2,8 @@ package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
@@ -9,17 +11,16 @@ import (
|
||||
)
|
||||
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
||||
// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer
|
||||
// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6.
|
||||
// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
|
||||
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
||||
"no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)),
|
||||
"no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
|
||||
Options: []asks.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
@@ -27,3 +28,147 @@ func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// The options of the restore ask (ADR 0537 §3, ADR 0539 §19), and what each does on the node.
|
||||
const (
|
||||
OptionRestore = "restore"
|
||||
OptionLeaveStopped = "leave-stopped"
|
||||
)
|
||||
|
||||
// RestoreAsk is the ask about a module holding data whose fingerprint failed its health check on a node (ADR 0429 §6): it is
|
||||
// not set back by itself, since a set-back would lose the writes made since. words is the check's words. Restore sets the
|
||||
// node back to the last healthy fingerprint and loses the newer writes; leaving it stopped keeps the data as it is.
|
||||
func RestoreAsk(key, node, module, fp, words string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s failed its health check on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s holds data, and its health check failed on %s at %s: %s. Setting it back to the version that ran healthy "+
|
||||
"would lose the writes made since; leaving it stopped keeps the data as it is.", module, node, short(fp), words),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionRestore, Label: "Restore it", Does: "the node is set back to the last healthy version and the newer writes are lost", Level: asks.Acknowledge},
|
||||
{ID: OptionLeaveStopped, Label: "Leave it stopped", Does: "the module stays as it is on the node and the data is kept", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module stays as it is on the node",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// HeldCondition is the condition the assigner raises when it holds a fingerprint of a module (ADR 0429 §6, ADR 0539 §19), or
|
||||
// clears (raised false) when the hold is released. It names the module, the node whose health check failed and the check's
|
||||
// words, under the assigner's own name (ADR 0437 clause 9), keyed by the held fingerprint. needsYou is true for a module holding
|
||||
// data, whose restore ask waits for the operator.
|
||||
func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".held." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
c := asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp))}
|
||||
what := fmt.Sprintf("The health check of %s failed on %s at %s: %s. The assigner holds that version and wants no node to move to it.", module, node, short(fp), words)
|
||||
if needsYou {
|
||||
c.Needs = "answer whether to restore " + module + " on " + node
|
||||
c.Explanation = asks.NeedsYou + c.Needs + ". " + what
|
||||
} else {
|
||||
c.Explanation = asks.NothingToDo + " " + what + " Nodes that wanted it are set back to the version that ran healthy."
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||
const (
|
||||
OptionYes = "yes"
|
||||
OptionNo = "no"
|
||||
OptionBackupAgain = delivery.BackupAgain
|
||||
OptionBackupSkip = delivery.BackupSkip
|
||||
)
|
||||
|
||||
// RecordAsk is the ask about a move of a module of policy record on node to the composition with the digest, judged by the
|
||||
// controller to recreate a container, to move data, or not to be judgeable (ADR 0539 §14). The want names the composition only
|
||||
// after the answer yes. The way the move was judged is in the ask.
|
||||
func RecordAsk(key, node, module, fp string, need RecordNeed, now time.Time) delivery.AssignerAsk {
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + need.Words,
|
||||
}[need.Judgement]
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s waits for your yes on %s", module, node),
|
||||
Explanation: fmt.Sprintf("%s is recorded: a person takes each of its moves. The next one on %s (version %s) %s. "+
|
||||
"The node is told to apply it only after you say yes.", module, node, short(fp), what),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionYes, Label: "Yes, move it", Does: "the node is told to apply the new composition of " + module, Level: asks.Acknowledge},
|
||||
{ID: OptionNo, Label: "No, not now", Does: "the node keeps what it runs", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// NextBackupAsking is the number of the asking that follows the one numbered after (a negative number: none yet). Numbers 0 and 1
|
||||
// give one id (delivery.AskingID), so the first asking is 0 and the next 2.
|
||||
func NextBackupAsking(after int) int {
|
||||
if after < 0 {
|
||||
return 0
|
||||
}
|
||||
return max(after+1, 2)
|
||||
}
|
||||
|
||||
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
|
||||
|
||||
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
|
||||
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
|
||||
// under a new number (ADR 0539 §16).
|
||||
func BackupAskDigest(a delivery.AssignerAsk) string {
|
||||
if m := backupDigest.FindStringSubmatch(a.Ask.Explanation); m != nil {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
|
||||
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
|
||||
func BackupAnswerWords(option string, asking int) string {
|
||||
return fmt.Sprintf("the operator answered %q to the backup ask (asking %d)", option, asking+1)
|
||||
}
|
||||
|
||||
// BackupAsk is the ask about a backup of a module on node that failed before an apply (ADR 0539 §16): back up again, or go on
|
||||
// without a backup. digest is the composition about to be applied and asking the number of the asking, 0 the first; words are the
|
||||
// backup's own words, from the node's have.
|
||||
func BackupAsk(key, node, module, fp, digest, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
Headline: fmt.Sprintf("The backup of %s failed on %s", module, node),
|
||||
Explanation: fmt.Sprintf("The backup before applying %s at %s on %s failed: %s. %s keeps running what it has until you answer. (composition %s)",
|
||||
module, short(fp), node, words, module, digest),
|
||||
Options: []asks.Option{
|
||||
{ID: OptionBackupAgain, Label: "Back up again", Does: "the node tries the backup again before it applies", Level: asks.Acknowledge},
|
||||
{ID: OptionBackupSkip, Label: "Go on without a backup", Does: "the node applies the new version without a backup", Level: asks.Acknowledge},
|
||||
},
|
||||
Who: asks.Operator,
|
||||
Expires: now.Add(7 * 24 * time.Hour),
|
||||
OnExpiry: "the module keeps running what it has",
|
||||
About: key,
|
||||
}}
|
||||
}
|
||||
|
||||
// BackupCondition is the condition raised when a backup before an apply fails and the module is held on the node (ADR 0539 §16),
|
||||
// naming the node, the module and the backup's words; cleared under the same key.
|
||||
func BackupCondition(node, module, fp, words string, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".backup." + node + "." + module + "." + strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared}
|
||||
}
|
||||
headline := fmt.Sprintf("The backup of %s failed on %s", module, node)
|
||||
needs := "answer whether to back up again or go on without a backup"
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". The backup before applying " + module + " at " + short(fp) + " on " + node + " failed: " + words + "."}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -26,3 +28,99 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3 (ADR 0539 §19): the ask about a data module held on a node is a valid ask of the router and of mesh-sdk's layout at
|
||||
// its restore key, of a kind the view may answer; it offers restoring (losing the newer writes) and leaving it stopped.
|
||||
func TestTheRestoreAskIsValid(t *testing.T) {
|
||||
key, err := delivery.RestoreAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := RestoreAsk(key, "ace", "mail", fNew, "connection refused", now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
var ids []string
|
||||
for _, o := range a.Ask.Options {
|
||||
ids = append(ids, o.ID)
|
||||
}
|
||||
if len(ids) != 2 || ids[0] != OptionRestore || ids[1] != OptionLeaveStopped {
|
||||
t.Errorf("options = %v", ids)
|
||||
}
|
||||
if !strings.Contains(a.Ask.Explanation, "connection refused") || !strings.Contains(a.Ask.Explanation, "ace") {
|
||||
t.Errorf("the ask does not name the node and the check's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
|
||||
// name (ADR 0437 clause 9), and is cleared under the same key.
|
||||
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Subject.ID != "mail" || c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") || !strings.Contains(c.Key, "mail") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := HeldCondition("mail", fNew, "", "", false, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
now := time.Now()
|
||||
rkey, err := delivery.RecordAskKey("ace", "mail", "sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := RecordAsk(rkey, "ace", "mail", fNew, RecordNeed{Node: "ace", Digest: "sha256:" + strings.Repeat("b", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"}, now)
|
||||
bkey, err := delivery.BackupAskKey("ace", "mail", fNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dgst := "sha256:" + strings.Repeat("b", 64)
|
||||
b := BackupAsk(bkey, "ace", "mail", fNew, dgst, "no space left", 0, now)
|
||||
for key, a := range map[string]delivery.AssignerAsk{rkey: r, bkey: b} {
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("%s: the router refuses the ask: %v", key, err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("%s: cannot be written: %v", key, err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("%s: the view may not answer it: %v", key, err)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(r.Ask.Explanation, "gone") || r.Ask.Options[0].ID != OptionYes {
|
||||
t.Errorf("the record ask = %+v", r.Ask)
|
||||
}
|
||||
if !strings.Contains(b.Ask.Explanation, "no space left") || b.Ask.Options[0].ID != delivery.BackupAgain || b.Ask.Options[1].ID != delivery.BackupSkip {
|
||||
t.Errorf("the backup ask = %+v", b.Ask)
|
||||
}
|
||||
// the id binds the ask to the composition and to its number (review round 1, items 3 and 4)
|
||||
if again := BackupAsk(bkey, "ace", "mail", fNew, dgst, "x", NextBackupAsking(0), now); again.Ask.ID == b.Ask.ID {
|
||||
t.Errorf("asked again under the same id")
|
||||
}
|
||||
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
|
||||
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
|
||||
}
|
||||
if BackupAskDigest(b) != dgst {
|
||||
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
|
||||
}
|
||||
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
|
||||
t.Errorf("two answers of the same option are one want value")
|
||||
}
|
||||
c := BackupCondition("ace", "mail", fNew, "no space left", true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its
|
||||
// assignment names, and that build's fingerprint of the new form (empty when its record names none).
|
||||
type Sent struct {
|
||||
Sends bool
|
||||
Build string
|
||||
FP string
|
||||
}
|
||||
|
||||
// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4),
|
||||
// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and
|
||||
// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the
|
||||
// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said
|
||||
// in words, for acceptance test 17 of design 47 to read.
|
||||
func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) {
|
||||
var d delivery.Difference
|
||||
removed := want != nil && want.Removed
|
||||
if want != nil && !removed {
|
||||
d.Want = want.FP
|
||||
}
|
||||
d.WantRemoved = removed
|
||||
if !sent.Sends {
|
||||
if want == nil || removed {
|
||||
return delivery.Difference{}, false
|
||||
}
|
||||
d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing"
|
||||
return d, true
|
||||
}
|
||||
d.Sent, d.SentBuild = sent.FP, sent.Build
|
||||
switch {
|
||||
case sent.FP == "":
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build)
|
||||
case removed:
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP))
|
||||
case sent.FP == d.Want:
|
||||
return delivery.Difference{}, false
|
||||
case hold.Rule != 0 && sent.FP == newest:
|
||||
d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words)
|
||||
case d.Want == "":
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP))
|
||||
default:
|
||||
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want))
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3.
|
||||
func TestADifferenceExplainedByTheCanary(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary}
|
||||
d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"])
|
||||
if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew {
|
||||
t.Fatalf("difference = %+v, %v; want rule 3", d, ok)
|
||||
}
|
||||
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||
t.Errorf("the difference cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written
|
||||
// unexplained, with words, and is writable.
|
||||
func TestCompare(t *testing.T) {
|
||||
w := delivery.Want{FP: fNew}
|
||||
if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok {
|
||||
t.Errorf("an equal want and send is a difference")
|
||||
}
|
||||
if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok {
|
||||
t.Errorf("no want and nothing sent is a difference")
|
||||
}
|
||||
removed := delivery.Want{Removed: true}
|
||||
if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok {
|
||||
t.Errorf("a removed want and nothing sent is a difference")
|
||||
}
|
||||
old := delivery.Want{FP: fOld}
|
||||
for name, c := range map[string]struct {
|
||||
want *delivery.Want
|
||||
sent Sent
|
||||
}{
|
||||
"sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}},
|
||||
"names no fingerprint": {&w, Sent{Sends: true, Build: "b"}},
|
||||
"sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
"sends nothing": {&w, Sent{}},
|
||||
"no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
"no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||
} {
|
||||
d, ok := Compare(c.want, c.sent, fNew, Holding{})
|
||||
if !ok || !d.Unexplained || d.Words == "" {
|
||||
t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok)
|
||||
continue
|
||||
}
|
||||
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||
t.Errorf("%s: the difference cannot be written: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,24 @@ type Module struct {
|
||||
Granted bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
Allowed func(node, digest string) bool
|
||||
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
||||
Released bool
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
||||
Restored func(node string) bool
|
||||
}
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if m.Composed == nil {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return m.Composed(node, fp)
|
||||
}
|
||||
|
||||
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
|
||||
@@ -72,6 +90,9 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
if !m.Granted {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
@@ -117,10 +138,24 @@ type Outcome struct {
|
||||
Wants map[string]delivery.Want
|
||||
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||
Held *delivery.Held
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why, for the comparison.
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
// back: the operator is asked whether to restore each (ADR 0539 §19).
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
Rules map[string]Holding
|
||||
}
|
||||
|
||||
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
||||
// controller's judgement of the move and its words.
|
||||
type RecordNeed struct {
|
||||
Node string
|
||||
Digest string
|
||||
Judgement delivery.Judgement
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why.
|
||||
type Holding struct {
|
||||
Rule int
|
||||
@@ -130,12 +165,12 @@ type Holding struct {
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||
// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A
|
||||
// module holding data is not set back but asked about (Restore). A
|
||||
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
||||
held := m.Held
|
||||
if !held && m.FP != "" {
|
||||
if !held && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
||||
out.Held = &delivery.Held{Node: node, Words: h.Detail}
|
||||
@@ -152,30 +187,57 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
before, had := wants[node]
|
||||
var next delivery.Want
|
||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||
if w, ok := m.userListRevision(node, before, had, n); ok {
|
||||
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch ok, rule, words := Holds(mm, node, n); {
|
||||
case ok:
|
||||
next = delivery.Want{FP: m.FP}
|
||||
d, _ := m.composition(node, m.FP)
|
||||
next = delivery.Want{FP: m.FP, Composed: d.Digest}
|
||||
if had && !before.Removed && before.FP == m.FP && before.Composed == d.Digest {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if need, wait := m.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
continue
|
||||
}
|
||||
case held && had && before.FP == m.FP:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
last := lastHealthy[node]
|
||||
switch {
|
||||
case keepsData(m.Manifest):
|
||||
case keepsData(m.Manifest) && (m.Restored == nil || !m.Restored(node)):
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP))
|
||||
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked whether to restore it", m.Name, short(m.FP))
|
||||
out.Restore = append(out.Restore, node)
|
||||
case last == "":
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
|
||||
default:
|
||||
next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words}
|
||||
d, there := m.composition(node, last)
|
||||
if !there {
|
||||
next = before
|
||||
next.Words = fmt.Sprintf("%s is held at %s, and the composition of %s at %s is gone, so %s is not set back", m.Name, short(m.FP), m.Name, short(last), node)
|
||||
break
|
||||
}
|
||||
next = delivery.Want{FP: last, Composed: d.Digest, Newest: m.FP, Rule: rule, Words: words}
|
||||
}
|
||||
default:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
if !had || before.Removed {
|
||||
continue
|
||||
}
|
||||
next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words}
|
||||
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
|
||||
if before.FP == m.FP {
|
||||
next = delivery.Want{FP: m.FP}
|
||||
next = delivery.Want{FP: m.FP, Composed: before.Composed}
|
||||
}
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||
@@ -203,3 +265,67 @@ func short(fp string) string {
|
||||
}
|
||||
return part[:12]
|
||||
}
|
||||
|
||||
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
|
||||
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
|
||||
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
|
||||
// same fingerprint judged neither (a user list only, §7).
|
||||
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
|
||||
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
// Right after the switch no node has a readable want, and a have from the old node-engine names a fingerprint and no composition
|
||||
// (ADR 0539 §9, §11): a node running this fingerprint is given its want at it with no hold when the controller judges the composition
|
||||
// unchanged, equal to the declaration the node last applied (ADR 0540 §9), since the node agent then applies nothing. A composition that
|
||||
// differs from it (a changed setting or secret), or that the controller could not judge, waits for the ask. A new fingerprint still waits.
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
prev := before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
if m.Allowed != nil && m.Allowed(node, d.Digest) {
|
||||
return nil, false
|
||||
}
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
|
||||
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
if d.Policy != "" {
|
||||
return d.Policy == delivery.PolicyRecord
|
||||
}
|
||||
policy, _, _ := catalogue.DerivedUpgrade(m.Manifest)
|
||||
return policy == catalogue.PolicyRecord
|
||||
}
|
||||
|
||||
// userListRevision is the want that names the bus module's composition at the fingerprint the node runs when that composition
|
||||
// differs from the one the node runs only in the bus's user list (ComposedDigest.UserListOnly), and whether there is one. A node
|
||||
// whose want names another fingerprint than the one it runs, or a module that does not provide the bus, has none.
|
||||
func (m Module) userListRevision(node string, before delivery.Want, had bool, n Nodes) (delivery.Want, bool) {
|
||||
if !catalogue.ProvidesBus(m.Manifest) {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
if !ran || running.FP == "" || (had && (before.Removed || before.FP != running.FP)) {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
d, ok := m.composition(node, running.FP)
|
||||
if !ok || !d.UserListOnly {
|
||||
return delivery.Want{}, false
|
||||
}
|
||||
w := delivery.Want{FP: running.FP, Composed: d.Digest}
|
||||
if had && before.Composed == d.Digest {
|
||||
w.Backup, w.Words = before.Backup, before.Words
|
||||
}
|
||||
return w, true
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -18,7 +19,7 @@ var (
|
||||
// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider
|
||||
// store healthy on ace and the canary ace running fNew healthy.
|
||||
func ready() (Module, Nodes) {
|
||||
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Nodes: []string{"g14", "ace"},
|
||||
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "ace"},
|
||||
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
|
||||
n := Nodes{
|
||||
Online: map[string]bool{"ace": true, "g14": true},
|
||||
@@ -31,8 +32,16 @@ func ready() (Module, Nodes) {
|
||||
return m, n
|
||||
}
|
||||
|
||||
// dig is the digest of the composition at a fingerprint in these tests: the fingerprint's last character repeated.
|
||||
func dig(fp string) string { return "sha256:" + strings.Repeat(fp[len(fp)-1:], 64) }
|
||||
|
||||
// composedEverywhere is a Module.Composed with a composition for every node at every fingerprint, judged neither.
|
||||
func composedEverywhere(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, true
|
||||
}
|
||||
|
||||
func wantedOld() map[string]delivery.Want {
|
||||
return map[string]delivery.Want{"ace": {FP: fOld}, "g14": {FP: fOld}}
|
||||
return map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}, "g14": {FP: fOld, Composed: dig(fOld)}}
|
||||
}
|
||||
|
||||
// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so
|
||||
@@ -41,7 +50,7 @@ func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
|
||||
m, n := ready()
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 {
|
||||
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
|
||||
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
|
||||
}
|
||||
}
|
||||
@@ -59,6 +68,9 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||
},
|
||||
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
|
||||
delivery.RuleComposed: func(m *Module, _ *Nodes) {
|
||||
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
|
||||
},
|
||||
} {
|
||||
m, n := ready()
|
||||
breakIt(&m, &n)
|
||||
@@ -98,11 +110,11 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
|
||||
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}, "g14": {FP: fOld}}, map[string]string{"ace": fOld})
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
||||
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
|
||||
t.Fatalf("held = %+v", out.Held)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w)
|
||||
}
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
@@ -122,6 +134,20 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
|
||||
if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") {
|
||||
t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w)
|
||||
}
|
||||
// 1906/D3: and the node is to be asked whether to restore it
|
||||
if !slices.Equal(out.Restore, []string{"ace"}) {
|
||||
t.Errorf("restore asks = %v, want ace", out.Restore)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D3: a module without data is set back and asked about nothing.
|
||||
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
|
||||
if len(out.Restore) != 0 {
|
||||
t.Errorf("restore asks = %v", out.Restore)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that
|
||||
@@ -129,7 +155,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
|
||||
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
|
||||
m, n := ready()
|
||||
m.Nodes = []string{"ace"}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew}, "g14": {FP: fOld}}, nil)
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
||||
if w := out.Wants["g14"]; !w.Removed {
|
||||
t.Errorf("g14's want = %+v, want removed", w)
|
||||
}
|
||||
@@ -137,3 +163,360 @@ func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
|
||||
t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §6): the seventh rule. A want waits for the composition of the module for that node at F, and names its
|
||||
// digest once it exists; a node with no want before still gets none until then.
|
||||
func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
|
||||
m, n := ready()
|
||||
have := map[string]bool{}
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if !have[node+"|"+fp] {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
for node, w := range out.Wants {
|
||||
if w.FP == fNew {
|
||||
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
|
||||
}
|
||||
}
|
||||
if r := out.Rules["ace"]; r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "ace") {
|
||||
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
|
||||
}
|
||||
// and a node that never had a want gets none
|
||||
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
|
||||
t.Errorf("wants without a composition: %+v", out.Wants)
|
||||
}
|
||||
have["ace|"+fNew] = true
|
||||
out = Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
|
||||
}
|
||||
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||
t.Errorf("g14, whose composition does not exist, moved: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2 (ADR 0539 §5): a new revision of the composition at the same fingerprint reaches the node only through a new want,
|
||||
// by the same rules: written when they hold, and not while one does not.
|
||||
func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
|
||||
m, n := ready()
|
||||
revised := "sha256:" + strings.Repeat("9", 64)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedNeither}, true
|
||||
}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
|
||||
out := Wants(m, n, running, nil)
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
if w := out.Wants[node]; w.FP != fNew || w.Composed != revised {
|
||||
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
|
||||
}
|
||||
}
|
||||
m.Granted = false
|
||||
out = Wants(m, n, running, nil)
|
||||
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
|
||||
t.Errorf("a revision was written while a rule holds: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
|
||||
func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
if fp == fOld {
|
||||
return delivery.ComposedDigest{}, false
|
||||
}
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
out := Wants(m, n, running, map[string]string{"ace": fOld})
|
||||
if w := out.Wants["ace"]; w.FP == fOld {
|
||||
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
|
||||
}
|
||||
m.Composed = composedEverywhere
|
||||
out = Wants(m, n, running, map[string]string{"ace": fOld})
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
|
||||
t.Errorf("set-back want = %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// movingTo is a module ready to move, whose composition at F is judged j against the one it runs, and whose digest is dig(F).
|
||||
func movingTo(j delivery.Judgement) (Module, Nodes, map[string]delivery.Want) {
|
||||
m, n := ready()
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: j}
|
||||
if j != delivery.JudgedNeither {
|
||||
d.Against, d.Words = dig(fOld), "judged"
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
return m, n, wantedOld()
|
||||
}
|
||||
|
||||
func recording(m *Module) {
|
||||
m.Manifest.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "a person takes each build"}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §14): a module of policy record whose move is judged to recreate a container, to move data, or cannot be
|
||||
// judged, is not written until its ask is answered yes; the same at a new revision of the same fingerprint.
|
||||
func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T) {
|
||||
for _, j := range []delivery.Judgement{delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot} {
|
||||
m, n, wanted := movingTo(j)
|
||||
recording(&m)
|
||||
out := Wants(m, n, wanted, nil)
|
||||
if w, ok := out.Wants["ace"]; ok {
|
||||
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
|
||||
}
|
||||
var ace *RecordNeed
|
||||
for i := range out.Record {
|
||||
if out.Record[i].Node == "ace" {
|
||||
ace = &out.Record[i]
|
||||
}
|
||||
}
|
||||
if ace == nil || ace.Digest != dig(fNew) || ace.Judgement != j {
|
||||
t.Errorf("%s: the asks = %+v, want one for ace naming the digest and the judgement", j, out.Record)
|
||||
}
|
||||
// a new revision at the same fingerprint is asked too
|
||||
same := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}}
|
||||
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
|
||||
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
|
||||
}
|
||||
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
|
||||
out = Wants(m, n, wanted, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("%s: after the yes ace's want = %+v", j, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: a move judged neither is written at once, whatever the policy; so is a module the node never ran; and a module
|
||||
// that rolls (not record) is never asked.
|
||||
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
|
||||
m, n, wanted := movingTo(delivery.JudgedNeither)
|
||||
recording(&m)
|
||||
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("neither: %+v", out)
|
||||
}
|
||||
m, n, _ = movingTo(delivery.JudgedRecreates)
|
||||
recording(&m)
|
||||
delete(n.Haves["ace"], "mail")
|
||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("a node that never ran it: %+v", out)
|
||||
}
|
||||
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
|
||||
m, n, _ = movingTo(delivery.JudgedCannot)
|
||||
recording(&m)
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew || len(out.Record) == 0 {
|
||||
t.Errorf("a node that runs it with no want: %+v", out)
|
||||
}
|
||||
m, n, wanted = movingTo(delivery.JudgedRecreates)
|
||||
if out := Wants(m, n, wanted, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("a rolled module: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
|
||||
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
|
||||
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
|
||||
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
|
||||
}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
|
||||
out := Wants(m, n, running, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
|
||||
}
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
||||
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
|
||||
m, n := ready()
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew), Backup: delivery.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
|
||||
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the answer was erased: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §19): a hold released by the operator is not held again by a health check that failed before, and the
|
||||
// wants move by the rules again.
|
||||
func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "x"}
|
||||
m.Released = true
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleNotHeld}}, map[string]string{"ace": fOld})
|
||||
if out.Held != nil {
|
||||
t.Errorf("a released fingerprint was held again: %+v", out.Held)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) || w.Rule != 0 {
|
||||
t.Errorf("ace's want = %+v, want the held fingerprint written again", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: a data module whose restore ask the operator answered "restore" is set back like any other, losing the newer writes;
|
||||
// answered "leave stopped" (or not answered) it stays.
|
||||
func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
|
||||
m, n := ready()
|
||||
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}
|
||||
m.Restored = func(node string) bool { return false }
|
||||
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.FP == fOld {
|
||||
t.Errorf("set back without the operator's restore: %+v", w)
|
||||
}
|
||||
m.Restored = func(node string) bool { return node == "ace" }
|
||||
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) {
|
||||
t.Errorf("after restore ace's want = %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §14): the policy the controller holds beside the composition decides, the operator's own choice included;
|
||||
// a composition with none (made by hand) falls back to what the manifest derives.
|
||||
func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
|
||||
m, n, wanted := movingTo(delivery.JudgedMovesData)
|
||||
with := func(policy, from string) {
|
||||
inner := m.Composed
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
d, ok := inner(node, fp)
|
||||
d.Policy, d.PolicyFrom = policy, from
|
||||
return d, ok
|
||||
}
|
||||
}
|
||||
with(delivery.PolicyRecord, "person") // a roll module the operator set to record
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].FP == fNew {
|
||||
t.Errorf("the operator's record policy was not honoured: %+v", out)
|
||||
}
|
||||
recording(&m) // the manifest says record, the operator chose roll
|
||||
with("roll", "person")
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
|
||||
t.Errorf("the operator's roll policy was not honoured: %+v", out)
|
||||
}
|
||||
with("", "") // made by hand: the manifest decides
|
||||
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
|
||||
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
|
||||
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
|
||||
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
|
||||
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1, item 1 (ADR 0539 §7): a revision of the bus module's composition that differs only in its user list is written
|
||||
// at once, at the fingerprint the node runs, before the record hold, whatever the controller judges of the file's mount (it
|
||||
// says recreates) and whatever policy the bus has (record, from the bus). The shape is the controller's: judged recreates against
|
||||
// the composition the node runs, policy record, UserListOnly set.
|
||||
func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedRecreates)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
userList := "sha256:" + strings.Repeat("7", 64)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
d := delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "the user list file is mounted",
|
||||
Policy: delivery.PolicyRecord, PolicyFrom: "the bus"}
|
||||
if fp == fOld {
|
||||
d.Digest, d.UserListOnly = userList, true // the composition at the fingerprint the node runs, with the new user list
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
for name, wanted := range map[string]map[string]delivery.Want{
|
||||
"a want at the fingerprint it runs": {"ace": {FP: fOld, Composed: dig(fOld)}},
|
||||
"no readable want": nil,
|
||||
} {
|
||||
out := Wants(m, n, wanted, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != userList {
|
||||
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
|
||||
}
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("%s: the user-list revision was asked: %+v", name, out.Record)
|
||||
}
|
||||
}
|
||||
// the same flag on a module that is not the bus changes nothing
|
||||
m.Manifest.Provides = nil
|
||||
if out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
|
||||
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 2, N1 (ADR 0539 §9, §11): right after the switch no node has a readable want and every have comes from the old node-engine
|
||||
// with a fingerprint and no composition, so every move is judged cannot-judge. A node running F with such a have gets its want at F
|
||||
// with no record hold, for a record module and for the bus alike (the node agent applies nothing when the composition equals the
|
||||
// declaration it kept); a new fingerprint still waits for its ask.
|
||||
func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
|
||||
for name, bus := range map[string]bool{"a record module": false, "the bus": true} {
|
||||
// only a composition the controller judged unchanged (it equals the declaration the node last applied, ADR 0540 §9) skips the hold
|
||||
unchanged := func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedUnchanged}, true
|
||||
}
|
||||
m, n, _ := movingTo(delivery.JudgedCannot)
|
||||
recording(&m)
|
||||
m.Nodes = []string{"ace"}
|
||||
if bus {
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
}
|
||||
m.Composed = unchanged
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} // no composition named
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
|
||||
}
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("%s: asked at the switch: %+v", name, out.Record)
|
||||
}
|
||||
// a composition that differs from the declaration the node kept (a changed setting or secret), and one the controller did not judge,
|
||||
// still wait for the ask
|
||||
for what, j := range map[string]delivery.Judgement{"differing": delivery.JudgedRecreates, "unjudged": delivery.JudgedCannot, "judged neither": delivery.JudgedNeither} {
|
||||
m2, n2, _ := movingTo(j)
|
||||
recording(&m2)
|
||||
m2.Nodes = []string{"ace"}
|
||||
if bus {
|
||||
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
}
|
||||
n2.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m2, n2, nil, nil)
|
||||
if j == delivery.JudgedNeither {
|
||||
if out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("%s, %s: %+v", name, what, out)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if w, ok := out.Wants["ace"]; ok || len(out.Record) == 0 {
|
||||
t.Errorf("%s, %s: written at the switch with no yes: %+v %+v", name, what, w, out.Record)
|
||||
}
|
||||
}
|
||||
// a have that names a composition is judged as before
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP == fNew && len(out.Record) == 0 {
|
||||
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
|
||||
}
|
||||
// a new fingerprint still asks (the bus is barred instead)
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["ace"]; ok {
|
||||
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
|
||||
}
|
||||
if !bus && len(out.Record) == 0 {
|
||||
t.Errorf("%s: a new fingerprint was not asked", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
|
||||
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||
// its three states, and holds a credential of its own.
|
||||
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
func TestManifest(t *testing.T) {
|
||||
m := load(t)
|
||||
if m.Module != "mesh-delivery-assigner" {
|
||||
@@ -32,9 +32,9 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants",
|
||||
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||
}
|
||||
@@ -51,9 +51,10 @@ func TestManifest(t *testing.T) {
|
||||
if m.SecretsOwner != "mesh-delivery-assigner" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1622: the assigner puts its compose asks on the Go build seat, as the checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go", m.Uses)
|
||||
// 1906/D3: it says its conditions on the operator channel; 1622: it puts its compose asks on the Go build seat, as the
|
||||
// checker puts its checks.
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
}
|
||||
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
||||
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
||||
|
||||
@@ -6,16 +6,19 @@
|
||||
"mesh-build-go.steps",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-controller.grants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"node-engine.have"
|
||||
],
|
||||
"uses": [
|
||||
"node-build-go"
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "asks", "history": 5 },
|
||||
|
||||
@@ -121,5 +121,12 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifest([]byte(kept.JSON))
|
||||
return parseManifest(kept.JSON)
|
||||
}
|
||||
|
||||
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
|
||||
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
|
||||
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
|
||||
func parseManifest(raw string) (catalogue.Manifest, error) {
|
||||
return catalogue.ParseManifestForInputs([]byte(raw))
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
@@ -93,7 +92,7 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
|
||||
if mk, err := delivery.ManifestsKey(m.Manifest); err == nil {
|
||||
if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found {
|
||||
if kept, err := delivery.DecodeManifest(raw); err == nil {
|
||||
if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil {
|
||||
if man, err := parseManifest(kept.JSON); err == nil && man.Build != nil {
|
||||
for _, a := range man.Build.Artifacts {
|
||||
if a.Language != "" {
|
||||
set[a.Language] = true
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -112,3 +113,31 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t
|
||||
t.Errorf("another repository: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the
|
||||
// catalogue's predefined.json this process never holds: the indexer and the assigner read it with
|
||||
// catalogue.ParseManifestForInputs (issues 1751, 1906). xterm's module.json from mesh-catalog at 835bf22b.
|
||||
func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/catalogue-835bf22b-xterm.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest := string(raw)
|
||||
data, err := delivery.EncodeManifest(delivery.Manifest{Module: "xterm", JSON: manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := memManifests{delivery.ManifestDigest([]byte(manifest)): data}
|
||||
var said []string
|
||||
r := &repoModules{manifests: store, logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
|
||||
putModule(t, r, "xterm", "novox/mesh-catalog", manifest)
|
||||
r.BuildModules("novox/mesh-catalog")
|
||||
for _, s := range said {
|
||||
if strings.Contains(s, "cannot be read") {
|
||||
t.Errorf("the manifest was not read: %s", s)
|
||||
}
|
||||
}
|
||||
if _, err := parseManifest(manifest); err != nil {
|
||||
t.Errorf("manifestReader: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
{
|
||||
"module": "xterm",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"capabilities": [
|
||||
"package-manager"
|
||||
],
|
||||
"requires": [
|
||||
"x11-display"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-terminal-emulator",
|
||||
"scope": "node",
|
||||
"serves": [
|
||||
"open"
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"xterm_font",
|
||||
"xterm_colours"
|
||||
],
|
||||
"config": {
|
||||
"face": {
|
||||
"type": "string",
|
||||
"description": "The font family xterm draws with, as fc-list names it: JetBrainsMono Nerd Font, because research 026/04 chose it for its glyphs the prompt uses. A family the machine does not have falls back to the nearest installed face (xterm_font says which).",
|
||||
"pattern": "[A-Za-z0-9 ._-]{1,64}",
|
||||
"default": "JetBrainsMono Nerd Font"
|
||||
},
|
||||
"size": {
|
||||
"type": "number",
|
||||
"description": "The size of xterm's text, in points: 12 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.",
|
||||
"minimum": 6,
|
||||
"maximum": 72,
|
||||
"predefined": "font-size",
|
||||
"default": 12
|
||||
},
|
||||
"save-lines": {
|
||||
"type": "number",
|
||||
"description": "How many lines of scrollback xterm keeps per terminal: 10000 holds a long build log, and each line costs memory in every open terminal.",
|
||||
"minimum": 0,
|
||||
"maximum": 1000000,
|
||||
"default": 10000
|
||||
},
|
||||
"term-name": {
|
||||
"type": "string",
|
||||
"description": "The TERM value xterm sets for the programs it runs: xterm-256color, because the palette has 256 colours and every program knows the name.",
|
||||
"pattern": "[A-Za-z0-9._-]{1,64}",
|
||||
"default": "xterm-256color"
|
||||
},
|
||||
"render-font": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws with fontconfig faces (true) or the old X core fonts (false): true, because the face is a fontconfig family and core fonts cannot show it.",
|
||||
"default": true
|
||||
},
|
||||
"bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm draws bold text by overstriking the normal face (true): false, because the face has a real bold style and overstriking blurs it.",
|
||||
"default": false
|
||||
},
|
||||
"always-bold-mode": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm overstrikes bold even where the face has a bold style (true): false, for the same reason as bold-mode.",
|
||||
"default": false
|
||||
},
|
||||
"allow-bold-fonts": {
|
||||
"type": "boolean",
|
||||
"description": "Whether xterm may use the face's bold style for bold text (true): true, so bold text is drawn with the real bold weight.",
|
||||
"default": true
|
||||
},
|
||||
"background": {
|
||||
"type": "string",
|
||||
"description": "The terminal's background colour, as #rrggbb: black, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"foreground": {
|
||||
"type": "string",
|
||||
"description": "The terminal's text colour, as #rrggbb: a light grey, easier on the eyes than white on black.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a8a8a8"
|
||||
},
|
||||
"color0": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 0, the normal black, as #rrggbb: #000000, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#000000"
|
||||
},
|
||||
"color1": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 1, the normal red, as #rrggbb: #9e1828, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#9e1828"
|
||||
},
|
||||
"color2": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 2, the normal green, as #rrggbb: #008800, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#008800"
|
||||
},
|
||||
"color3": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 3, the normal yellow, as #rrggbb: #d2bb4b, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#d2bb4b"
|
||||
},
|
||||
"color4": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 4, the normal blue, as #rrggbb: #414171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#414171"
|
||||
},
|
||||
"color5": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 5, the normal magenta, as #rrggbb: #963c59, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#963c59"
|
||||
},
|
||||
"color6": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 6, the normal cyan, as #rrggbb: #418179, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#418179"
|
||||
},
|
||||
"color7": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 7, the normal white, as #rrggbb: #bebebe, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bebebe"
|
||||
},
|
||||
"color8": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 8, the bright black, as #rrggbb: #666666, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#666666"
|
||||
},
|
||||
"color9": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 9, the bright red, as #rrggbb: #bc5766, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#bc5766"
|
||||
},
|
||||
"color10": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 10, the bright green, as #rrggbb: #61a171, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#61a171"
|
||||
},
|
||||
"color11": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 11, the bright yellow, as #rrggbb: #e7db52, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#e7db52"
|
||||
},
|
||||
"color12": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 12, the bright blue, as #rrggbb: #5085af, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#5085af"
|
||||
},
|
||||
"color13": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 13, the bright magenta, as #rrggbb: #a97a99, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#a97a99"
|
||||
},
|
||||
"color14": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 14, the bright cyan, as #rrggbb: #6ba4a4, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#6ba4a4"
|
||||
},
|
||||
"color15": {
|
||||
"type": "string",
|
||||
"description": "Palette entry 15, the bright white, as #rrggbb: #ffffff, as both workstations ran it.",
|
||||
"pattern": "#[0-9a-fA-F]{6}",
|
||||
"default": "#ffffff"
|
||||
},
|
||||
"copy-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that copies the selection to the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> C, so a plain Ctrl+C still interrupts the program. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> C"
|
||||
},
|
||||
"paste-binding": {
|
||||
"type": "string",
|
||||
"description": "The key combination that pastes the clipboard, in X translation syntax: Ctrl Shift ~Alt <Key> V, the pair of copy-binding. It holds no colon, backslash or line break, which would end the resource.",
|
||||
"pattern": "[^:\\\\\\n]+",
|
||||
"default": "Ctrl Shift ~Alt <Key> V"
|
||||
}
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"collection": "environment",
|
||||
"name": "terminal",
|
||||
"attributes": {
|
||||
"kind": "variable",
|
||||
"name": "TERMINAL"
|
||||
},
|
||||
"template": "files/env-terminal"
|
||||
},
|
||||
{
|
||||
"collection": "shell:xresources:normal",
|
||||
"name": "xresources-normal",
|
||||
"template": "files/xresources-normal"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "xterm"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/xterm-tools",
|
||||
"binary": "xterm-tools",
|
||||
"loads": [
|
||||
"xterm-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"mesh-delivery-view/internal/view"
|
||||
@@ -82,9 +83,50 @@ func Tools(r view.Reader) []stdio.Tool {
|
||||
commit, _ := args["commit"].(string)
|
||||
return r.BuilderBuild(module, fp2, commit)
|
||||
}},
|
||||
{Name: view.WantsReadTool,
|
||||
Description: "The assigner's wants and holds as stored in mesh-delivery-assigner_wants: wants by key (want.<node>.<module>) and " +
|
||||
"held fingerprints by key (held.<module>.<hex>). Narrow by node and module (a hold has no node and is shown when no node is given). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Wants(node, module)
|
||||
}},
|
||||
{Name: view.WantReadTool,
|
||||
Description: "One want of the assigner as stored: what the node is to run of the module (fingerprint, the newest it is held back from and the rule, removed, words). Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"node", "module"}, "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Want(node, module)
|
||||
}},
|
||||
{Name: view.DifferencesReadTool,
|
||||
Description: "The assigner's differences between a want and what the controller sends, as stored in mesh-delivery-assigner_differences, " +
|
||||
"by key (<node>.<module>), with the rule that explains each or unexplained. Narrow by node and module. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Differences(node, module)
|
||||
}},
|
||||
{Name: view.HavesReadTool,
|
||||
Description: "What each node has, as the node-engine stored it in node-engine_have: by key <node>.<module> (the fingerprint it runs, its phase and health), " +
|
||||
"and <node>.node (the node's own facts). Narrow by node and module. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
return r.Haves(node, module)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
var nodeModule = map[string]any{
|
||||
"node": map[string]any{"type": "string", "description": "only this node's keys"},
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}
|
||||
|
||||
func nodeModuleOf(args map[string]any) (string, string) {
|
||||
node, _ := args["node"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
return strings.TrimSpace(node), strings.TrimSpace(module)
|
||||
}
|
||||
|
||||
func read(state stdio.KeptState) view.StateRead {
|
||||
return view.StateRead{Keys: state.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
e, err := state.Get(key)
|
||||
@@ -102,7 +144,8 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
}
|
||||
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates"))}
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have"))}
|
||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -264,3 +264,68 @@ func TestCandidateToolsAnswerTheStoredValue(t *testing.T) {
|
||||
t.Errorf("by module = %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D1 AC1: assigner_wants, assigner_want, assigner_differences and node_haves answer as stored, narrowed by node and module.
|
||||
func keptReader() view.Reader {
|
||||
st := func(f fake) view.StateRead { return f.reader().State }
|
||||
return view.Reader{
|
||||
WantsState: st(fake{
|
||||
"want.ace.zsh": `{"fp":"` + fp2A + `"}`,
|
||||
"want.novox.zsh": `{"fp":"` + fp2B + `"}`,
|
||||
"want.ace.netcheck": `{"fp":"` + fp2A + `","rule":3}`,
|
||||
"held." + keyB: `{"node":"ace","words":"failed"}`,
|
||||
}),
|
||||
DifferencesState: st(fake{"ace.zsh": `{"unexplained":true}`, "novox.zsh": `{"rule":3}`}),
|
||||
HavesState: st(fake{"ace.zsh": `{"fp":"x"}`, "ace.node": `{"facts":1}`, "novox.zsh": `{"fp":"y"}`}),
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignerWantsToolAnswersAsStoredAndNarrows(t *testing.T) {
|
||||
r := keptReader()
|
||||
out, err := run(t, r, view.WantsReadTool, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(view.WantsAnswer)
|
||||
if len(a.Wants) != 3 || len(a.Held) != 1 || string(a.Wants["want.ace.zsh"]) != `{"fp":"`+fp2A+`"}` {
|
||||
t.Errorf("all = %+v", a)
|
||||
}
|
||||
out, _ = run(t, r, view.WantsReadTool, map[string]any{"node": "ace"})
|
||||
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
|
||||
t.Errorf("node ace = %v", got)
|
||||
}
|
||||
out, _ = run(t, r, view.WantsReadTool, map[string]any{"module": "zsh"})
|
||||
if got := out.(view.WantsAnswer).Wants; len(got) != 2 {
|
||||
t.Errorf("module zsh = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignerWantToolAnswersOneAndSaysWhenThereIsNone(t *testing.T) {
|
||||
r := keptReader()
|
||||
out, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "zsh"})
|
||||
if err != nil || string(out.(json.RawMessage)) != `{"fp":"`+fp2A+`"}` {
|
||||
t.Errorf("want = %v %v", out, err)
|
||||
}
|
||||
if _, err := run(t, r, view.WantReadTool, map[string]any{"node": "ace", "module": "gone"}); err == nil || !strings.Contains(err.Error(), "no want") {
|
||||
t.Errorf("a missing want: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignerDifferencesAndNodeHavesToolsAnswerAsStored(t *testing.T) {
|
||||
r := keptReader()
|
||||
out, err := run(t, r, view.DifferencesReadTool, map[string]any{"node": "novox"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := out.(view.KeptAnswer).Keys; len(got) != 1 || string(got["novox.zsh"]) != `{"rule":3}` {
|
||||
t.Errorf("differences = %v", got)
|
||||
}
|
||||
out, _ = run(t, r, view.HavesReadTool, map[string]any{"module": "zsh"})
|
||||
if got := out.(view.KeptAnswer).Keys; len(got) != 2 {
|
||||
t.Errorf("haves of zsh = %v", got)
|
||||
}
|
||||
out, _ = run(t, r, view.HavesReadTool, map[string]any{"node": "ace"})
|
||||
if got := out.(view.KeptAnswer).Keys; len(got) != 2 || got["ace.node"] == nil {
|
||||
t.Errorf("haves of ace (the node's own key too) = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -433,17 +433,17 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerDifferencesBucket, History: delivery.AssignerDifferencesHistory})
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerWantsBucket, History: delivery.AssignerWantsHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
const fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
for key, d := range map[string]delivery.Difference{
|
||||
"ace.plex": {Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"},
|
||||
"g14.plex": {Want: fpOld, Sent: fpNew, Unexplained: true},
|
||||
for key, d := range map[string]delivery.Want{
|
||||
"want.ace.plex": {FP: fpNew, Composed: "sha256:" + strings.Repeat("c", 64)},
|
||||
"want.g14.plex": {FP: fpOld, Composed: "sha256:" + strings.Repeat("c", 64), Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"},
|
||||
} {
|
||||
v, err := delivery.EncodeDifference(d)
|
||||
v, err := delivery.EncodeWant(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -465,13 +465,13 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC
|
||||
t.Fatalf("%s; last: %+v", what, last)
|
||||
}
|
||||
wait(func(s view.Snapshot) bool {
|
||||
d := s.Differences
|
||||
return len(d) == 2 && d[0].Node == "ace" && d[0].Rule == delivery.RuleCanary && d[1].Node == "g14" && d[1].Unexplained
|
||||
}, "the two differences never showed with their rules")
|
||||
if err := kv.Delete(ctx, "g14.plex"); err != nil {
|
||||
d := s.Wants
|
||||
return len(d) == 2 && d[0].Node == "ace" && d[0].State == view.WantPending && d[1].Node == "g14" && d[1].State == view.WantHeldBack && d[1].Rule == delivery.RuleCanary
|
||||
}, "the two wants never showed with where they stand")
|
||||
if err := kv.Delete(ctx, "want.g14.plex"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait(func(s view.Snapshot) bool { return len(s.Differences) == 1 }, "the difference taken away never left the list")
|
||||
wait(func(s view.Snapshot) bool { return len(s.Wants) == 1 }, "the want taken away never left the list")
|
||||
}
|
||||
|
||||
// TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof starts the program whole, with the assigner's asks, the view's
|
||||
@@ -696,3 +696,230 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC
|
||||
}
|
||||
t.Fatalf("the backlog never showed three waiting asks with the oldest's age; last: %+v", last.Backlog)
|
||||
}
|
||||
|
||||
// 1906/D9, whole on the bus: when the wants and the haves make the source revision delivered, the view says it once, as its event
|
||||
// `delivered` with the revision, the group and the heads, which the issue tracker derives "fix delivered" from.
|
||||
func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) {
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, name := range []string{delivery.SourceBucket, delivery.ModulesBucket, delivery.AssignerWantsBucket, delivery.HaveBucket} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[name] = kv
|
||||
}
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
fp := "src2:" + strings.Repeat("2", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
good, _ := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}})
|
||||
for bucket, kv := range map[string][]byte{delivery.ModulesBucket: mod} {
|
||||
if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, "want.ace.web", want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := kvs[delivery.SourceBucket].Put(ctx, delivery.SourceMainKey, src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
if info, err := events.Info(ctx); err != nil || info.State.Msgs != 0 {
|
||||
t.Fatalf("an event was said before the revision was delivered: %v %+v", err, info.State)
|
||||
}
|
||||
if _, err := kvs[delivery.HaveBucket].Put(ctx, "ace.web", good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var msg *jetstream.RawStreamMsg
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && msg == nil; time.Sleep(100 * time.Millisecond) {
|
||||
msg, _ = events.GetLastMsgForSubject(ctx, "mesh.mod.mesh-delivery-view.event."+DeliveredEventType)
|
||||
}
|
||||
if msg == nil {
|
||||
t.Fatal("the delivery of the source revision was never said")
|
||||
}
|
||||
var e view.DeliveredEvent
|
||||
if err := json.Unmarshal(msg.Data, &e); err != nil || e.Revision != rev || e.Group != "feat/x" || e.Heads["novox/web"] != strings.Repeat("1", 40) {
|
||||
t.Errorf("the event = %+v, %v; want revision %d, group feat/x and the heads", e, err, rev)
|
||||
}
|
||||
if msg.Header.Get("x-source") != "mesh-delivery-view" || msg.Header.Get("x-event-id") == "" {
|
||||
t.Errorf("the headers = %v", msg.Header)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D10, whole on the bus (through run): a want that differs from its have is said `moving`, the have reaching it running and
|
||||
// healthy `settled`, a have held or failed `stuck`, each once with the body the messenger reads and an event id per type, node, module
|
||||
// and composition.
|
||||
func TestProcessWholeSaysModulesInMotion(t *testing.T) {
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "VIEWEVENTS", Subjects: []string{"mesh.mod.mesh-delivery-view.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, name := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[name] = kv
|
||||
}
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
time.Sleep(2 * time.Second) // the view has read what stood: nothing
|
||||
|
||||
fpOld, fpNew := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64)
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
put := func(bucket, key string, v []byte, err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kvs[bucket].Put(ctx, key, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want := func(node, module string) {
|
||||
v, err := delivery.EncodeWant(delivery.Want{FP: fpNew, Composed: digest})
|
||||
put(delivery.AssignerWantsBucket, "want."+node+"."+module, v, err)
|
||||
}
|
||||
have := func(node, module, fp string, phase delivery.Phase, health delivery.Health, detail string) {
|
||||
v, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: phase, Health: health, Detail: detail})
|
||||
put(delivery.HaveBucket, node+"."+module, v, err)
|
||||
}
|
||||
have("ace", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "")
|
||||
want("ace", "web")
|
||||
have("g14", "web", fpOld, delivery.PhaseRunning, delivery.HealthOK, "")
|
||||
want("g14", "web")
|
||||
|
||||
said := func() map[string]jetstream.RawStreamMsg {
|
||||
out := map[string]jetstream.RawStreamMsg{}
|
||||
for seq := uint64(1); ; seq++ {
|
||||
m, err := events.GetMsg(ctx, seq)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
out[m.Header.Get("x-event-id")] = *m
|
||||
}
|
||||
}
|
||||
waitFor := func(id string) jetstream.RawStreamMsg {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if m, ok := said()[id]; ok {
|
||||
return m
|
||||
}
|
||||
}
|
||||
t.Fatalf("never said %s; said %v", id, func() []string {
|
||||
var ids []string
|
||||
for k := range said() {
|
||||
ids = append(ids, k)
|
||||
}
|
||||
return ids
|
||||
}())
|
||||
return jetstream.RawStreamMsg{}
|
||||
}
|
||||
hex := strings.Repeat("c", 64)
|
||||
mv := waitFor("moving.ace.web." + hex)
|
||||
var body view.MotionBody
|
||||
if err := json.Unmarshal(mv.Data, &body); err != nil || body.Node != "ace" || body.Module != "web" || body.FromFP != fpOld || body.ToFP != fpNew || body.Composed != digest || body.BoundSeconds != view.MotionBoundSeconds || body.Since.IsZero() {
|
||||
t.Errorf("moving = %+v, %v", body, err)
|
||||
}
|
||||
if mv.Subject != "mesh.mod.mesh-delivery-view.event.moving" {
|
||||
t.Errorf("subject = %s", mv.Subject)
|
||||
}
|
||||
have("ace", "web", fpNew, delivery.PhaseRunning, delivery.HealthOK, "")
|
||||
st := waitFor("settled.ace.web." + hex)
|
||||
if err := json.Unmarshal(st.Data, &body); err != nil || body.Outcome != "running" {
|
||||
t.Errorf("settled = %+v, %v", body, err)
|
||||
}
|
||||
have("g14", "web", fpOld, delivery.PhaseHeld, delivery.HealthWaits, "no space left")
|
||||
stk := waitFor("stuck.g14.web." + hex)
|
||||
if err := json.Unmarshal(stk.Data, &body); err != nil || body.Outcome != "stopped" || body.Words != "no space left" {
|
||||
t.Errorf("stuck = %+v, %v", body, err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
count := map[string]int{}
|
||||
for id := range said() {
|
||||
count[strings.SplitN(id, ".", 2)[0]]++
|
||||
}
|
||||
if count["moving"] != 2 || count["settled"] != 1 || count["stuck"] != 1 {
|
||||
t.Errorf("events said by type = %v, want moving 2, settled 1, stuck 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,8 +138,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
model.SetWriter(kvWriter{ctx: ctx, js: js})
|
||||
model.SetMotionEvent(func(e view.MotionEvent) {
|
||||
if err := emitEvent(ctx, js, cred, e.Type, e.Body, e.ID); err != nil {
|
||||
logger.Printf("the %s of %s on %s could not be said: %v", e.Type, e.Body.Module, e.Body.Node, err)
|
||||
}
|
||||
})
|
||||
model.SetDeliveredEvent(func(e view.DeliveredEvent) {
|
||||
if err := emitEvent(ctx, js, cred, DeliveredEventType, e, fmt.Sprintf("delivered.%d", e.Revision)); err != nil {
|
||||
logger.Printf("the delivery of source revision %d could not be said: %v", e.Revision, err)
|
||||
}
|
||||
})
|
||||
model.SetBacklog(view.BacklogOver(js, 5*time.Second))
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} {
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.ControllerComposedBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket} {
|
||||
go follow(ctx, js, bucket, model, logger)
|
||||
}
|
||||
select {
|
||||
@@ -204,10 +214,36 @@ func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, model
|
||||
return errors.New("the watch ended")
|
||||
}
|
||||
if e == nil { // the last values have all been delivered
|
||||
model.Initial(bucket)
|
||||
continue
|
||||
}
|
||||
model.Apply(view.Entry{Bucket: bucket, Key: e.Key(), Value: e.Value(), At: e.Created(),
|
||||
Deleted: e.Operation() != jetstream.KeyValuePut})
|
||||
Deleted: e.Operation() != jetstream.KeyValuePut, Revision: e.Revision()})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeliveredEventType is the event the view says when a source revision is delivered: mesh.mod.mesh-delivery-view.event.delivered
|
||||
// (issue 1906 D9, ADR 0539 §17). Its body is view.DeliveredEvent.
|
||||
const DeliveredEventType = "delivered"
|
||||
|
||||
// emitEvent publishes an event of the view's own namespace, mesh.mod.<module>.event.<type>, with the SDK's envelope headers
|
||||
// (as mesh-sdk's process.Conn.Emit does) and the id as the bus's own dedup id, so the same revision said twice in the stream's
|
||||
// duplicate window is kept once.
|
||||
func emitEvent(ctx context.Context, js jetstream.JetStream, cred credential, eventType string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + eventType)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// diffEntry is a key of the assigner's differences as the assigner writes it (mesh-sdk go/delivery EncodeDifference).
|
||||
func differenceE(t *testing.T, node, module string, d delivery.Difference) Entry {
|
||||
t.Helper()
|
||||
key, err := delivery.DifferenceKey(node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := delivery.EncodeDifference(d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AssignerDifferencesBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func TestEveryDifferenceIsListedWithItsRule(t *testing.T) { // 1297/B8 AC
|
||||
m := New()
|
||||
fill(m,
|
||||
differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}),
|
||||
differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}),
|
||||
differenceE(t, "ace", "old", delivery.Difference{WantRemoved: true, Sent: fpOld, Rule: delivery.RuleNotHeld}),
|
||||
)
|
||||
got := m.Snapshot().Differences
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("differences = %+v, want all three", got)
|
||||
}
|
||||
// ordered by node then module
|
||||
if got[0].Node != "ace" || got[0].Module != "old" || got[1].Module != "plex" || got[2].Node != "g14" {
|
||||
t.Errorf("order = %+v", got)
|
||||
}
|
||||
if r := got[1]; r.Want != fpOld || r.Sent != fpNew || r.Rule != delivery.RuleCanary || r.Unexplained || r.Words != "ace is not the canary" {
|
||||
t.Errorf("row = %+v", r)
|
||||
}
|
||||
if !got[0].WantRemoved || got[0].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("removed row = %+v", got[0])
|
||||
}
|
||||
if !got[2].Unexplained || got[2].Rule != 0 {
|
||||
t.Errorf("unexplained row = %+v", got[2])
|
||||
}
|
||||
// a difference taken away (the want and the send agree again) leaves the list
|
||||
fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "g14.plex", Deleted: true})
|
||||
if got := m.Snapshot().Differences; len(got) != 2 {
|
||||
t.Errorf("after the delete = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableDifferenceIsNamedAndNotDropped(t *testing.T) { // never dropped silently
|
||||
m := New()
|
||||
fill(m, Entry{Bucket: delivery.AssignerDifferencesBucket, Key: "ace.plex", Value: []byte(`{"v":99}`), At: t0})
|
||||
s := m.Snapshot()
|
||||
if len(s.Differences) != 0 || len(s.Unreadable) != 1 || s.Unreadable[0].Bucket != delivery.AssignerDifferencesBucket || s.Unreadable[0].Key != "ace.plex" {
|
||||
t.Errorf("snapshot = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableAndPageShowTheWantsAgainstTheSends(t *testing.T) { // 1297/B8 AC
|
||||
m := New()
|
||||
fill(m,
|
||||
differenceE(t, "ace", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Rule: delivery.RuleCanary, Words: "ace is not the canary"}),
|
||||
differenceE(t, "g14", "plex", delivery.Difference{Want: fpOld, Sent: fpNew, Unexplained: true}),
|
||||
)
|
||||
var out strings.Builder
|
||||
Table(&out, m.Snapshot())
|
||||
for _, w := range []string{"WANTS AGAINST SENDS", "ace", "plex", fpOld, fpNew, "rule 3", "ace is not the canary", "unexplained"} {
|
||||
if !strings.Contains(out.String(), w) {
|
||||
t.Errorf("table lacks %q:\n%s", w, out.String())
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/", "/wants"} {
|
||||
rec := httptest.NewRecorder()
|
||||
Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
body := rec.Body.String()
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%s = %d", path, rec.Code)
|
||||
}
|
||||
for _, w := range []string{"WANTS AGAINST SENDS", "rule 3", "unexplained", "ace is not the canary"} {
|
||||
if !strings.Contains(body, w) {
|
||||
t.Errorf("%s lacks %q:\n%s", path, w, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
// With none, the page says so and does not hide the heading.
|
||||
rec := httptest.NewRecorder()
|
||||
Web(New()).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/wants", nil))
|
||||
if !strings.Contains(rec.Body.String(), "no difference") {
|
||||
t.Errorf("empty page = %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,10 @@ type Reader struct {
|
||||
State StateRead // the indexer's modules
|
||||
Builds StateRead // the builder's builds
|
||||
|
||||
CandidatesState StateRead // the indexer's candidates
|
||||
CandidatesState StateRead // the indexer's candidates
|
||||
WantsState StateRead // the assigner's wants and holds
|
||||
DifferencesState StateRead // the assigner's differences
|
||||
HavesState StateRead // the node-engine's haves
|
||||
}
|
||||
|
||||
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
||||
@@ -328,3 +331,124 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
|
||||
}
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// The read tools of the assigner's wants and differences and of the node-engine's haves (issue 1906, D1): the keys of those
|
||||
// buckets answered as stored, so what the assigner wants, what it found different and what each node has can be read live.
|
||||
// They write nothing.
|
||||
const (
|
||||
WantsReadTool = "assigner_wants"
|
||||
WantReadTool = "assigner_want"
|
||||
DifferencesReadTool = "assigner_differences"
|
||||
HavesReadTool = "node_haves"
|
||||
)
|
||||
|
||||
// WantsAnswer is the answer of assigner_wants: the wants by key (`want.<node>.<module>`) and the holds by key
|
||||
// (`held.<module>.<hex>`), each as stored, and the keys that are neither.
|
||||
type WantsAnswer struct {
|
||||
Wants map[string]json.RawMessage `json:"wants"`
|
||||
Held map[string]json.RawMessage `json:"held"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
}
|
||||
|
||||
// KeptAnswer is the answer of assigner_differences and node_haves: the keys of the bucket as stored, and the keys that
|
||||
// are no `<node>.<module>` key.
|
||||
type KeptAnswer struct {
|
||||
Keys map[string]json.RawMessage `json:"keys"`
|
||||
Unreadable []string `json:"unreadable,omitempty"`
|
||||
}
|
||||
|
||||
func (s StateRead) all(what string) (map[string]json.RawMessage, error) {
|
||||
if s.Keys == nil || s.Get == nil {
|
||||
return nil, fmt.Errorf("%s cannot be read here", what)
|
||||
}
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := map[string]json.RawMessage{}
|
||||
for _, k := range keys {
|
||||
raw, err := s.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Wants is every want and hold of the assigner, narrowed by node and module (a hold has no node and is kept when the module
|
||||
// matches).
|
||||
func (r Reader) Wants(node, module string) (WantsAnswer, error) {
|
||||
all, err := r.WantsState.all("the assigner's wants")
|
||||
if err != nil {
|
||||
return WantsAnswer{}, err
|
||||
}
|
||||
out := WantsAnswer{Wants: map[string]json.RawMessage{}, Held: map[string]json.RawMessage{}}
|
||||
for k, raw := range all {
|
||||
if n, m, err := delivery.ParseWantKey(k); err == nil {
|
||||
if (node == "" || n == node) && (module == "" || m == module) {
|
||||
out.Wants[k] = raw
|
||||
}
|
||||
continue
|
||||
}
|
||||
if m, _, err := delivery.ParseHeldKey(k); err == nil {
|
||||
if node == "" && (module == "" || m == module) {
|
||||
out.Held[k] = raw
|
||||
}
|
||||
continue
|
||||
}
|
||||
out.Unreadable = append(out.Unreadable, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Want is the stored want of module on node.
|
||||
func (r Reader) Want(node, module string) (json.RawMessage, error) {
|
||||
key, err := delivery.WantKey(strings.TrimSpace(node), strings.TrimSpace(module))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.WantsState.Get == nil {
|
||||
return nil, fmt.Errorf("the assigner's wants cannot be read here")
|
||||
}
|
||||
raw, err := r.WantsState.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("the assigner has no want %s", key)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
func (r Reader) kept(s StateRead, what, node, module string) (KeptAnswer, error) {
|
||||
all, err := s.all(what)
|
||||
if err != nil {
|
||||
return KeptAnswer{}, err
|
||||
}
|
||||
out := KeptAnswer{Keys: map[string]json.RawMessage{}}
|
||||
for k, raw := range all {
|
||||
n, m, isNode, err := delivery.ParseHaveKey(k)
|
||||
if err != nil {
|
||||
out.Unreadable = append(out.Unreadable, k)
|
||||
continue
|
||||
}
|
||||
if (node == "" || n == node) && (module == "" || (!isNode && m == module)) {
|
||||
out.Keys[k] = raw
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Differences is the assigner's differences as stored, narrowed by node and module.
|
||||
func (r Reader) Differences(node, module string) (KeptAnswer, error) {
|
||||
return r.kept(r.DifferencesState, "the assigner's differences", node, module)
|
||||
}
|
||||
|
||||
// Haves is the node-engine's haves (and each node's own key) as stored, narrowed by node and module.
|
||||
func (r Reader) Haves(node, module string) (KeptAnswer, error) {
|
||||
return r.kept(r.HavesState, "the haves", node, module)
|
||||
}
|
||||
|
||||
@@ -209,33 +209,42 @@ func askText(s Snapshot) [][]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// differenceText is one line per difference: the node, the module, what the assigner wants, what the controller sends, the rule
|
||||
// that explains it (or unexplained) and the assigner's words.
|
||||
func differenceText(s Snapshot) [][]string {
|
||||
// wantText is one line per want: the node, the module, what the assigner wants, what the node runs, where the two stand and the
|
||||
// assigner's words (the rule that holds a node back, when one does).
|
||||
func wantText(s Snapshot) [][]string {
|
||||
var out [][]string
|
||||
for _, d := range s.Differences {
|
||||
want, sent := d.Want, d.Sent
|
||||
switch {
|
||||
case d.WantRemoved:
|
||||
want = "removed"
|
||||
case want == "":
|
||||
for _, w := range s.Wants {
|
||||
want, have := w.Want, w.Have
|
||||
if want == "" {
|
||||
want = "none"
|
||||
}
|
||||
if sent == "" {
|
||||
sent = "none"
|
||||
if d.SentBuild != "" {
|
||||
sent = "build " + d.SentBuild
|
||||
}
|
||||
if have == "" {
|
||||
have = "none"
|
||||
}
|
||||
why := "unexplained"
|
||||
if !d.Unexplained {
|
||||
why = fmt.Sprintf("rule %d", d.Rule)
|
||||
words := w.Words
|
||||
if w.Rule != 0 {
|
||||
words = fmt.Sprintf("rule %d: %s", w.Rule, w.Words)
|
||||
}
|
||||
out = append(out, []string{d.Node, d.Module, want, sent, why, d.Words})
|
||||
judged := w.Judgement
|
||||
if judged == "" {
|
||||
judged = "none"
|
||||
} else if w.Judged != "" {
|
||||
words = strings.TrimSpace(words + " " + w.Judged)
|
||||
}
|
||||
out = append(out, []string{w.Node, w.Module, want, have, w.State, judged, words})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sourceText is the source revision's derived state in one line, then one line for each module on a node it waits for.
|
||||
func sourceText(s Snapshot) []string {
|
||||
if s.Source == nil {
|
||||
return nil
|
||||
}
|
||||
out := []string{fmt.Sprintf("source revision %d: %s", s.Source.Revision, s.Source.State)}
|
||||
return append(out, s.Source.Waiting...)
|
||||
}
|
||||
|
||||
// backlogText is one line per language and kind: its waiting asks, the asks held by a node, the oldest waiting ask's age.
|
||||
func backlogText(s Snapshot) [][]string {
|
||||
var out [][]string
|
||||
@@ -254,7 +263,7 @@ func backlogText(s Snapshot) [][]string {
|
||||
|
||||
var backlogHead = []string{"LANGUAGE", "KIND", "WAITING", "HELD", "OLDEST WAITING"}
|
||||
|
||||
var differenceHead = []string{"NODE", "MODULE", "WANT", "SENT", "RULE", "WORDS"}
|
||||
var wantHead = []string{"NODE", "MODULE", "WANT", "HAVE", "STATE", "MOVE JUDGED", "WORDS"}
|
||||
|
||||
var head = []string{"NODE", "MODULE", "ASSIGNED", "HAVE", "PHASE", "HEALTH", "SINCE", "DETAIL", "WAITS"}
|
||||
|
||||
@@ -303,10 +312,16 @@ func Table(w io.Writer, s Snapshot) {
|
||||
row(cells)
|
||||
}
|
||||
}
|
||||
if d := differenceText(s); len(d) > 0 {
|
||||
if src := sourceText(s); len(src) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"WANTS AGAINST SENDS"})
|
||||
row(differenceHead)
|
||||
for _, line := range src {
|
||||
row([]string{line})
|
||||
}
|
||||
}
|
||||
if d := wantText(s); len(d) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"WANTS AGAINST HAVES"})
|
||||
row(wantHead)
|
||||
for _, cells := range d {
|
||||
row(cells)
|
||||
}
|
||||
@@ -356,16 +371,16 @@ var page = template.Must(template.New("p").Parse(`<!doctype html><html><head><me
|
||||
{{if .Groups}}<h2>CANDIDATES</h2><table><tr><th>GROUP</th><th>PULL REQUEST</th><th>HEAD</th><th>MODULES</th></tr>{{range .Groups}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Asks}}<h2>ASKS</h2><table><tr><th>ASK</th><th>HEADLINE</th><th>WORDS</th><th>PULL REQUESTS</th><th>ANSWER</th></tr>{{range .Asks}}<tr>{{range .Cells}}<td>{{.}}</td>{{end}}<td>{{if .Options}}<form method="post" action="/answer"><input type="hidden" name="key" value="{{.Key}}"><input name="words" placeholder="words (optional)">{{range .Options}} <button name="option" value="{{.}}">{{.}}</button>{{end}}</form>{{end}}</td></tr>{{end}}</table>{{end}}
|
||||
{{if .Backlog}}<h2>BACKLOG OF BUILD ASKS</h2><table><tr>{{range .BacklogHead}}<th>{{.}}</th>{{end}}</tr>{{range .Backlog}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Differences}}<h2>WANTS AGAINST SENDS</h2><table><tr>{{range .DifferenceHead}}<th>{{.}}</th>{{end}}</tr>{{range .Differences}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Source}}<h2>SOURCE REVISION</h2><p>{{range .Source}}{{.}}<br>{{end}}</p>{{end}}{{if .Wants}}<h2>WANTS AGAINST HAVES</h2><table><tr>{{range .WantHead}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Fingerprints}}<h2>FINGERPRINTS</h2><table><tr><th>MODULE</th><th>STATE</th><th>INDEXER AGAINST BUILD</th></tr>{{range .Fingerprints}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Offline}}<h2>OFFLINE</h2><table>{{range .Offline}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Stuck}}<h2>STUCK</h2><table><tr><th>BUCKET</th><th>KEY</th><th>WAITED FOR</th><th>SINCE</th></tr>{{range .Stuck}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Unreadable}}<h2>UNREADABLE</h2><table>{{range .Unreadable}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}</body></html>`))
|
||||
|
||||
var wantsPage = template.Must(template.New("w").Parse(`<!doctype html><html><head><meta charset="utf-8"><meta http-equiv="refresh" content="5">
|
||||
<title>Wants against sends</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
|
||||
<body><p><a href="/">What each node runs</a></p><h1>WANTS AGAINST SENDS</h1>
|
||||
{{if .Differences}}<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>{{range .Differences}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
|
||||
<title>Wants against haves</title><style>body{font:14px sans-serif;margin:1em}table{border-collapse:collapse}td,th{padding:2px 10px;text-align:left}</style></head>
|
||||
<body><p><a href="/">What each node runs</a></p><h1>WANTS AGAINST HAVES</h1>
|
||||
{{if .Wants}}<table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>{{range .Wants}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
|
||||
{{else}}<p>no difference: every want is what the controller sends.</p>{{end}}</body></html>`))
|
||||
|
||||
// Web is the read-only web view of the same rows, refreshed by the browser.
|
||||
@@ -387,7 +402,7 @@ func Web(m *Model) http.Handler {
|
||||
}
|
||||
if r.URL.Path == "/wants" {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
wantsPage.Execute(w, map[string]any{"Head": differenceHead, "Differences": differenceText(m.Snapshot())})
|
||||
wantsPage.Execute(w, map[string]any{"Head": wantHead, "Wants": wantText(m.Snapshot())})
|
||||
return
|
||||
}
|
||||
if r.URL.Path != "/" {
|
||||
@@ -410,7 +425,7 @@ func Web(m *Model) http.Handler {
|
||||
for _, u := range snap.Unreadable {
|
||||
bad = append(bad, []string{u.Bucket, u.Key, u.Error})
|
||||
}
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Differences": differenceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "DifferenceHead": differenceHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askPage(snap), "Fingerprints": fingerprintText(snap), "Wants": wantText(snap), "Source": sourceText(snap), "Backlog": backlogText(snap), "BacklogHead": backlogHead, "WantHead": wantHead, "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ type Entry struct {
|
||||
Value []byte
|
||||
At time.Time
|
||||
Deleted bool
|
||||
// Revision is the bucket revision of the change, when the watch gives one (the source revision is the gate's source key's).
|
||||
Revision uint64
|
||||
}
|
||||
|
||||
// Row is one module on one node.
|
||||
@@ -66,27 +68,59 @@ type Snapshot struct {
|
||||
Fingerprints []FingerprintRow `json:"fingerprints"`
|
||||
// Asks is every ask the gate holds that has no answer yet: what waits for the operator's Approve.
|
||||
Asks []AskRow `json:"asks"`
|
||||
// Differences is every difference between what the assigner wants a node to run and what the controller sends, with the rule that
|
||||
// explains it or unexplained (novox/hq ADR 0528 clause 4, issue 1297 B8).
|
||||
Differences []DifferenceRow `json:"differences"`
|
||||
// Wants is every want of the assigner beside the have of its node and where it stands (novox/hq ADR 0539 §8, issue 1906 D7).
|
||||
Wants []WantRow `json:"wants"`
|
||||
// Source is the state of the mesh's source revision, derived from the wants and the haves (ADR 0429 §1): recorded by no part.
|
||||
// Nil when the gate holds no source yet.
|
||||
Source *SourceRow `json:"source,omitempty"`
|
||||
// Backlog is, for each language and kind of build ask with one waiting or held, the asks waiting for a node, the asks a node
|
||||
// holds and the oldest waiting ask's age (issue 1887 A8a).
|
||||
Backlog []BacklogRow `json:"backlog"`
|
||||
}
|
||||
|
||||
// DifferenceRow is a module on a node whose want and send differ: the fingerprint wanted (empty when the node has no want,
|
||||
// WantRemoved when the want is removed), the fingerprint of the build the controller's assignment names (empty when it sends
|
||||
// none; SentBuild names that build), and the rule that explains it (Rule 1 to 6) or Unexplained.
|
||||
type DifferenceRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Want string `json:"want,omitempty"`
|
||||
WantRemoved bool `json:"want-removed,omitempty"`
|
||||
Sent string `json:"sent,omitempty"`
|
||||
SentBuild string `json:"sent-build,omitempty"`
|
||||
Rule int `json:"rule,omitempty"`
|
||||
Unexplained bool `json:"unexplained,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
// The states of a want against the have of its node.
|
||||
const (
|
||||
WantApplied = "applied" // the node runs the want's fingerprint, running and healthy
|
||||
WantPending = "pending" // the node has not yet run the want's fingerprint, or not healthy yet
|
||||
WantFailed = "failed" // the node runs the want's fingerprint and its health check failed
|
||||
WantHeldBack = "held back" // the assigner holds the node back from the newest fingerprint (Rule), or set it back
|
||||
WantRemoved = "removed" // the node's want is removed: the module is no longer assigned to it
|
||||
)
|
||||
|
||||
// WantRow is a want of the assigner beside the have of its node: the fingerprint wanted, the one the node runs with its phase and
|
||||
// health, the state of the two, and for a want held back the newest fingerprint, the rule and the assigner's words.
|
||||
type WantRow struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Want string `json:"want,omitempty"`
|
||||
Have string `json:"have,omitempty"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
Health string `json:"health,omitempty"`
|
||||
State string `json:"state"`
|
||||
Newest string `json:"newest,omitempty"`
|
||||
Rule int `json:"rule,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
// Judgement is the controller's judgement of the move to the composition the want names (recreates, moves-data, neither or
|
||||
// cannot-judge), and Judged its words; empty when the controller wrote none (ADR 0539 §14).
|
||||
Judgement string `json:"judgement,omitempty"`
|
||||
Judged string `json:"judged,omitempty"`
|
||||
}
|
||||
|
||||
// The states of a source revision (ADR 0429 §1).
|
||||
const (
|
||||
SourceDelivered = "delivered"
|
||||
SourceNotDelivered = "not delivered"
|
||||
)
|
||||
|
||||
// SourceRow is the mesh's source revision (the revision of the gate's source key) and whether it is delivered: every want equals its
|
||||
// module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is assigned to
|
||||
// has a want. Waiting names each module on a node that is not there yet, and why.
|
||||
type SourceRow struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
State string `json:"state"`
|
||||
Waiting []string `json:"waiting,omitempty"`
|
||||
// Wants is how many wants of modules at the indexer were looked at; a revision with none is delivered of nothing.
|
||||
Wants int `json:"wants"`
|
||||
}
|
||||
|
||||
// AskRow is an open ask (novox/hq ADR 0437 §10): the gate's ask with no answer under its key in the teller's answers.
|
||||
@@ -178,22 +212,29 @@ type UnreadableRow struct {
|
||||
|
||||
// Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values.
|
||||
type Model struct {
|
||||
mu sync.Mutex
|
||||
assigns map[string]assignment
|
||||
builds map[string]buildEntry // by <module>.build.<id>
|
||||
groups map[string]groupEntry // the indexer's candidates, by group
|
||||
modules map[string]moduleEntry // the indexer's modules, by module
|
||||
keys map[string]keyEntry // the builder's build keys, by <module>.<fingerprint>
|
||||
asks map[string]askRecord // the gate's asks, by key
|
||||
diffs map[string]diffEntry // the assigner's differences, by <node>.<module>
|
||||
aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key
|
||||
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
||||
writer Writer
|
||||
backlog *Backlog
|
||||
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
||||
newest map[string]string // module -> id
|
||||
haves map[string]haveEntry
|
||||
nodes map[string]nodeEntry
|
||||
mu sync.Mutex
|
||||
assigns map[string]assignment
|
||||
builds map[string]buildEntry // by <module>.build.<id>
|
||||
groups map[string]groupEntry // the indexer's candidates, by group
|
||||
modules map[string]moduleEntry // the indexer's modules, by module
|
||||
keys map[string]keyEntry // the builder's build keys, by <module>.<fingerprint>
|
||||
asks map[string]askRecord // the gate's asks, by key
|
||||
wants map[string]wantEntry // the assigner's wants, by <node>.<module>
|
||||
source *sourceEntry // the gate's source key
|
||||
deliveredEvent func(DeliveredEvent)
|
||||
motionEvent func(MotionEvent)
|
||||
told map[string]bool // the motion events said, by their ids
|
||||
initial map[string]bool // the buckets whose watch handed over what stood at its start
|
||||
said uint64 // the source revision whose delivery was said
|
||||
composed map[string]composedEntry // the controller's digests and judgements, by <node>.<module>.<hex>
|
||||
aasks map[string]assignerAskRecord // the assigner's asks (hold, restore), by key
|
||||
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
||||
writer Writer
|
||||
backlog *Backlog
|
||||
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
||||
newest map[string]string // module -> id
|
||||
haves map[string]haveEntry
|
||||
nodes map[string]nodeEntry
|
||||
// the watchman's buckets, by key
|
||||
presence map[string]presenceEntry
|
||||
stuck map[string]stuckEntry
|
||||
@@ -256,9 +297,22 @@ type keyEntry struct {
|
||||
err string
|
||||
}
|
||||
|
||||
type diffEntry struct {
|
||||
value delivery.Difference
|
||||
type wantEntry struct {
|
||||
value delivery.Want
|
||||
err string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
type composedEntry struct {
|
||||
value delivery.ComposedDigest
|
||||
err string
|
||||
}
|
||||
|
||||
type sourceEntry struct {
|
||||
revision uint64
|
||||
group string
|
||||
heads map[string]string
|
||||
err string
|
||||
}
|
||||
|
||||
type assignerAskRecord struct {
|
||||
@@ -290,18 +344,64 @@ type nodeEntry struct {
|
||||
|
||||
// New is an empty model.
|
||||
func New() *Model {
|
||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, diffs: map[string]diffEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{},
|
||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{},
|
||||
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
||||
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
|
||||
}
|
||||
|
||||
// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed.
|
||||
// Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed. When the change makes
|
||||
// the source revision delivered, the event of it is said once (SetDeliveredEvent), after the model's lock is released.
|
||||
func (m *Model) Apply(e Entry) {
|
||||
m.mu.Lock()
|
||||
m.apply(e)
|
||||
event := m.deliveredNow()
|
||||
say := m.deliveredEvent
|
||||
motion, sayMotion := m.motionNow(e), m.motionEvent
|
||||
m.mu.Unlock()
|
||||
if event != nil && say != nil {
|
||||
say(*event)
|
||||
}
|
||||
if sayMotion != nil {
|
||||
for _, ev := range motion {
|
||||
sayMotion(ev)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeliveredEvent is what the view says when the source revision is delivered (event `delivered` of the view, issue 1906 D9, ADR
|
||||
// 0539 §17): the revision of the gate's source key, the group that moved it and every repository's merged head at it. A
|
||||
// consumer takes every earlier revision as covered by it, and takes the same revision twice as once (after a restart of the view).
|
||||
type DeliveredEvent struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
Group string `json:"group"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
}
|
||||
|
||||
// SetDeliveredEvent sets where the event of a delivered source revision goes; nothing is said until it is set.
|
||||
func (m *Model) SetDeliveredEvent(say func(DeliveredEvent)) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.deliveredEvent = say
|
||||
}
|
||||
|
||||
// deliveredNow is the event of the source revision when it is delivered now and has not been said, or nil. A revision that went
|
||||
// bad after it was delivered and recovered is not said again.
|
||||
func (m *Model) deliveredNow() *DeliveredEvent {
|
||||
if m.deliveredEvent == nil || m.source == nil || m.source.err != "" || m.said == m.source.revision {
|
||||
return nil
|
||||
}
|
||||
row := m.sourceRow()
|
||||
if row == nil || row.State != SourceDelivered || row.Wants == 0 {
|
||||
return nil
|
||||
}
|
||||
m.said = m.source.revision
|
||||
return &DeliveredEvent{Revision: m.source.revision, Group: m.source.group, Heads: m.source.heads}
|
||||
}
|
||||
|
||||
func (m *Model) apply(e Entry) {
|
||||
switch e.Bucket {
|
||||
case delivery.HaveBucket:
|
||||
node, module, isNode, err := delivery.ParseHaveKey(e.Key)
|
||||
node, _, isNode, err := delivery.ParseHaveKey(e.Key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -314,7 +414,7 @@ func (m *Model) Apply(e Entry) {
|
||||
m.nodes[node] = nodeEntry{node: n, err: errText(err), at: e.At}
|
||||
return
|
||||
}
|
||||
key := node + "." + module
|
||||
key := e.Key // the bucket's own key: a module name with a dot is escaped in it (issue 1150)
|
||||
if e.Deleted {
|
||||
delete(m.haves, key)
|
||||
return
|
||||
@@ -385,13 +485,38 @@ func (m *Model) Apply(e Entry) {
|
||||
if a, err := delivery.DecodeViewAnswer(e.Value); err == nil {
|
||||
m.chosen[askKey] = a
|
||||
}
|
||||
case delivery.AssignerDifferencesBucket:
|
||||
if e.Deleted {
|
||||
delete(m.diffs, e.Key)
|
||||
case delivery.AssignerWantsBucket:
|
||||
node, module, err := delivery.ParseWantKey(e.Key)
|
||||
if err != nil {
|
||||
return // a hold, and the node's own key, are not a module's want
|
||||
}
|
||||
key, kerr := delivery.HaveKey(node, module)
|
||||
if kerr != nil {
|
||||
return
|
||||
}
|
||||
d, err := delivery.DecodeDifference(e.Value)
|
||||
m.diffs[e.Key] = diffEntry{value: d, err: errText(err)}
|
||||
if e.Deleted {
|
||||
delete(m.wants, key)
|
||||
return
|
||||
}
|
||||
w, err := delivery.DecodeWant(e.Value)
|
||||
m.wants[key] = wantEntry{value: w, err: errText(err), at: e.At}
|
||||
case delivery.ControllerComposedBucket:
|
||||
if e.Deleted {
|
||||
delete(m.composed, e.Key)
|
||||
return
|
||||
}
|
||||
d, err := delivery.DecodeComposedDigest(e.Value)
|
||||
m.composed[e.Key] = composedEntry{value: d, err: errText(err)}
|
||||
case delivery.SourceBucket:
|
||||
if e.Key != delivery.SourceMainKey {
|
||||
return
|
||||
}
|
||||
if e.Deleted {
|
||||
m.source = nil
|
||||
return
|
||||
}
|
||||
src, err := delivery.DecodeSource(e.Value)
|
||||
m.source = &sourceEntry{revision: e.Revision, group: src.Group, heads: src.Heads, err: errText(err)}
|
||||
case delivery.AnswersBucket:
|
||||
if e.Deleted {
|
||||
delete(m.answers, e.Key)
|
||||
@@ -487,7 +612,17 @@ func (m *Model) snapshot() Snapshot {
|
||||
r.RequiredBy = a.RequiredBy
|
||||
}
|
||||
var waits []string
|
||||
h, has := m.haves[key]
|
||||
hk, herr := delivery.HaveKey(a.Node, a.Module)
|
||||
if herr != nil {
|
||||
hk = key
|
||||
}
|
||||
h, has := m.haves[hk]
|
||||
// The controller no longer writes the build of an assignment (R6): the row is the want and the have of the node.
|
||||
w, wanted := m.wants[hk]
|
||||
wanted = wanted && w.err == "" && !w.value.Removed && w.value.FP != ""
|
||||
if wanted {
|
||||
r.Assigned = w.value.FP
|
||||
}
|
||||
switch {
|
||||
case !has:
|
||||
waits = append(waits, "no have yet")
|
||||
@@ -495,8 +630,15 @@ func (m *Model) snapshot() Snapshot {
|
||||
r.Since, r.Error = h.at, h.err
|
||||
default:
|
||||
r.Have, r.Phase, r.Health = h.have.Build, string(h.have.Phase), string(h.have.Health)
|
||||
if r.Have == "" {
|
||||
r.Have = h.have.FP
|
||||
}
|
||||
r.Since, r.Detail = h.at, h.have.Detail
|
||||
if a.Build != "" && h.have.Build != a.Build {
|
||||
if wanted {
|
||||
if h.have.FP != w.value.FP {
|
||||
waits = append(waits, "the node has not applied "+w.value.FP)
|
||||
}
|
||||
} else if a.Build != "" && h.have.Build != a.Build {
|
||||
waits = append(waits, "the node has not applied "+a.Build)
|
||||
}
|
||||
}
|
||||
@@ -509,10 +651,13 @@ func (m *Model) snapshot() Snapshot {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
for key, h := range m.haves {
|
||||
if a, ok := m.assigns[key]; ok && a.open() {
|
||||
node, module, _, perr := delivery.ParseHaveKey(key)
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
if a, ok := m.assigns[node+"."+module]; ok && a.open() {
|
||||
continue
|
||||
}
|
||||
node, module, _, _ := delivery.ParseHaveKey(key)
|
||||
r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err}
|
||||
if h.err == "" && h.have.Phase == delivery.PhaseRemoved {
|
||||
// The node-engine says it took the module away: plainly removed, not left behind.
|
||||
@@ -520,6 +665,9 @@ func (m *Model) snapshot() Snapshot {
|
||||
}
|
||||
if h.err == "" {
|
||||
r.Have, r.Phase, r.Health, r.Detail = h.have.Build, string(h.have.Phase), string(h.have.Health), h.have.Detail
|
||||
if r.Have == "" {
|
||||
r.Have = h.have.FP
|
||||
}
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
@@ -585,9 +733,9 @@ func (m *Model) snapshot() Snapshot {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerAsksBucket, Key: key, Error: a.err})
|
||||
}
|
||||
}
|
||||
for key, d := range m.diffs {
|
||||
if d.err != "" {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerDifferencesBucket, Key: key, Error: d.err})
|
||||
for key, w := range m.wants {
|
||||
if w.err != "" {
|
||||
unreadable = append(unreadable, UnreadableRow{Bucket: delivery.AssignerWantsBucket, Key: key, Error: w.err})
|
||||
}
|
||||
}
|
||||
sort.Slice(offline, func(i, j int) bool {
|
||||
@@ -601,7 +749,7 @@ func (m *Model) snapshot() Snapshot {
|
||||
return unreadable[i].Bucket+"."+unreadable[i].Key < unreadable[j].Bucket+"."+unreadable[j].Key
|
||||
})
|
||||
return Snapshot{Rows: rows, Nodes: nodes, Offline: offline, Stuck: stuck, Unreadable: unreadable,
|
||||
Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Differences: m.differenceRows()}
|
||||
Groups: m.groupRows(), Fingerprints: m.fingerprintRows(), Asks: m.askRows(), Wants: m.wantRows(), Source: m.sourceRow()}
|
||||
}
|
||||
|
||||
// groupRows are the readable candidate groups, ordered by group, their pull requests by repository and number.
|
||||
@@ -840,20 +988,38 @@ func (m *Model) Answer(askKey, option, words string) error {
|
||||
return w.Put(key, body)
|
||||
}
|
||||
|
||||
// differenceRows are the readable differences, ordered by node and module.
|
||||
func (m *Model) differenceRows() []DifferenceRow {
|
||||
out := []DifferenceRow{}
|
||||
for key, d := range m.diffs {
|
||||
if d.err != "" {
|
||||
// wantRows are the readable wants beside the haves of their nodes, ordered by node and module.
|
||||
func (m *Model) wantRows() []WantRow {
|
||||
out := []WantRow{}
|
||||
for key, w := range m.wants {
|
||||
if w.err != "" {
|
||||
continue
|
||||
}
|
||||
node, module, err := delivery.ParseDifferenceKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
node, module := splitKey(key)
|
||||
v := w.value
|
||||
row := WantRow{Node: node, Module: module, Want: v.FP, Newest: v.Newest, Rule: v.Rule, Words: v.Words}
|
||||
h, has := m.haves[key]
|
||||
if has && h.err == "" {
|
||||
row.Have, row.Phase, row.Health = h.have.FP, string(h.have.Phase), string(h.have.Health)
|
||||
}
|
||||
v := d.value
|
||||
out = append(out, DifferenceRow{Node: node, Module: module, Want: v.Want, WantRemoved: v.WantRemoved, Sent: v.Sent, SentBuild: v.SentBuild,
|
||||
Rule: v.Rule, Unexplained: v.Unexplained, Words: v.Words})
|
||||
if ckey, err := delivery.ComposedKey(node, module, v.FP); err == nil && !v.Removed {
|
||||
if c, ok := m.composed[ckey]; ok && c.err == "" {
|
||||
row.Judgement, row.Judged = string(c.value.Judgement), c.value.Words
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case v.Removed:
|
||||
row.State = WantRemoved
|
||||
case v.Rule != 0 || v.Newest != "":
|
||||
row.State = WantHeldBack
|
||||
case row.Have == v.FP && h.have.Health == delivery.HealthFailed:
|
||||
row.State = WantFailed
|
||||
case row.Have == v.FP && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK:
|
||||
row.State = WantApplied
|
||||
default:
|
||||
row.State = WantPending
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Node != out[j].Node {
|
||||
@@ -863,3 +1029,257 @@ func (m *Model) differenceRows() []DifferenceRow {
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// sourceRow derives the state of the source revision the gate holds (ADR 0429 §1; recorded by no part): delivered when every want
|
||||
// equals its module's fingerprint at the indexer and every have equals its want, running and healthy, and every node a module is
|
||||
// assigned to has a want. It is nil when the gate holds no readable source.
|
||||
func (m *Model) sourceRow() *SourceRow {
|
||||
if m.source == nil || m.source.err != "" {
|
||||
return nil
|
||||
}
|
||||
waiting := []string{}
|
||||
short := func(fp string) string {
|
||||
fp = strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
if len(fp) > 12 {
|
||||
fp = fp[:12]
|
||||
}
|
||||
return fp
|
||||
}
|
||||
looked := 0
|
||||
for key, w := range m.wants {
|
||||
node, module := splitKey(key)
|
||||
mod, ok := m.modules[module]
|
||||
if w.err != "" || w.value.Removed || !ok || mod.err != "" || mod.value.FP2 == "" {
|
||||
continue
|
||||
}
|
||||
looked++
|
||||
h := m.haves[key]
|
||||
switch {
|
||||
case w.value.FP != mod.value.FP2:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s wants %s, not the module's %s", module, node, short(w.value.FP), short(mod.value.FP2)))
|
||||
case h.err != "" || h.have.FP != w.value.FP:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s does not yet run %s", module, node, short(w.value.FP)))
|
||||
case h.have.Health == delivery.HealthFailed:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s runs %s and its health check failed", module, node, short(w.value.FP)))
|
||||
case h.have.Phase != delivery.PhaseRunning || h.have.Health != delivery.HealthOK:
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s runs %s, not yet running and healthy", module, node, short(w.value.FP)))
|
||||
}
|
||||
}
|
||||
for key, a := range m.assigns {
|
||||
if !a.open() {
|
||||
continue
|
||||
}
|
||||
wk, werr := delivery.HaveKey(a.Node, a.Module)
|
||||
if werr != nil {
|
||||
wk = key
|
||||
}
|
||||
if _, ok := m.wants[wk]; ok {
|
||||
continue
|
||||
}
|
||||
if mod, ok := m.modules[a.Module]; ok && mod.err == "" && mod.value.FP2 != "" {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s has no want yet", a.Module, a.Node))
|
||||
}
|
||||
}
|
||||
sort.Strings(waiting)
|
||||
row := &SourceRow{Revision: m.source.revision, State: SourceDelivered, Wants: looked}
|
||||
if len(waiting) > 0 {
|
||||
row.State, row.Waiting = SourceNotDelivered, waiting
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
// splitKey is the node and the module (unescaped, issue 1150) of a model key, which is the have bucket's key form.
|
||||
func splitKey(key string) (node, module string) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
node, module, _ = strings.Cut(key, ".")
|
||||
}
|
||||
return node, module
|
||||
}
|
||||
|
||||
// The events of a module in motion (issue 1906 D10, for the messenger, ADR 0308): with no plan to say it, the view says that a
|
||||
// module's want on a node differs from its have (moving), that the have reached the want running and healthy (settled), or that it
|
||||
// is held or failed or stopped moving past the watchman's bound (stuck). Each is said once for a node, a module and a composition.
|
||||
const (
|
||||
MotionMoving = "moving"
|
||||
MotionSettled = "settled"
|
||||
MotionStuck = "stuck"
|
||||
// MotionBoundSeconds is how long a node may take to apply a want before the watchman says it stuck (watch.StepBound).
|
||||
MotionBoundSeconds = 1800
|
||||
)
|
||||
|
||||
// MotionBody is the body of the three events. Outcome and Words are those of settled and stuck: settled is "running"; stuck is
|
||||
// "failed" (a health check failed) or "stopped" (the module is held, or the watchman says the want stopped moving past its bound),
|
||||
// with the have's own words or the watchman's in Words.
|
||||
type MotionBody struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
FromFP string `json:"from_fp"`
|
||||
ToFP string `json:"to_fp"`
|
||||
Composed string `json:"composed"`
|
||||
Since time.Time `json:"since"`
|
||||
BoundSeconds int `json:"bound_seconds"`
|
||||
// At is when the have or the want last changed, the time the event is about.
|
||||
At time.Time `json:"at"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
// MotionEvent is an event to say: its type (moving, settled or stuck), its id (the bus's dedup id and the event id header, one for
|
||||
// a type, a node, a module and a composition) and its body.
|
||||
type MotionEvent struct {
|
||||
Type string
|
||||
ID string
|
||||
Body MotionBody
|
||||
}
|
||||
|
||||
// SetMotionEvent sets where the events of modules in motion go; nothing is said until it is set.
|
||||
func (m *Model) SetMotionEvent(say func(MotionEvent)) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.motionEvent = say
|
||||
}
|
||||
|
||||
// motionNow is the events to say now about the node and module the change touched: a want, a have, or the watchman's stuck key of a
|
||||
// want.
|
||||
func (m *Model) motionNow(e Entry) []MotionEvent {
|
||||
if m.motionEvent == nil || !m.motionReady() {
|
||||
return nil
|
||||
}
|
||||
var hk string
|
||||
switch e.Bucket {
|
||||
case delivery.AssignerWantsBucket:
|
||||
node, module, err := delivery.ParseWantKey(e.Key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
hk, _ = delivery.HaveKey(node, module)
|
||||
case delivery.HaveBucket:
|
||||
if _, _, isNode, err := delivery.ParseHaveKey(e.Key); err != nil || isNode {
|
||||
return nil
|
||||
}
|
||||
hk = e.Key
|
||||
case delivery.StuckBucket:
|
||||
st, ok := m.stuck[e.Key]
|
||||
if !ok || st.err != "" || st.value.Bucket != delivery.AssignerWantsBucket {
|
||||
return nil
|
||||
}
|
||||
node, module, err := delivery.ParseWantKey(st.value.Key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
hk, _ = delivery.HaveKey(node, module)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
return m.motionOf(hk)
|
||||
}
|
||||
|
||||
// motionReady says whether the view has read what stands in the buckets motion is derived from (the wants, the haves and the
|
||||
// watchman's stuck keys): before, a want read ahead of its have would be said moving though the node runs it.
|
||||
func (m *Model) motionReady() bool {
|
||||
for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
|
||||
if !m.initial[b] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Initial tells the model that the watch of a bucket has handed over every key that stood at its start. When the buckets motion is
|
||||
// derived from have all done so, the events of the modules in motion at that moment are said (the ids deduplicate what a restart
|
||||
// of the view says again).
|
||||
func (m *Model) Initial(bucket string) {
|
||||
m.mu.Lock()
|
||||
was := m.motionReady()
|
||||
m.initial[bucket] = true
|
||||
var out []MotionEvent
|
||||
if !was && m.motionReady() && m.motionEvent != nil {
|
||||
keys := make([]string, 0, len(m.wants))
|
||||
for k := range m.wants {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
out = append(out, m.motionOf(k)...)
|
||||
}
|
||||
}
|
||||
say := m.motionEvent
|
||||
m.mu.Unlock()
|
||||
for _, ev := range out {
|
||||
say(ev)
|
||||
}
|
||||
}
|
||||
|
||||
// motionOf is the events to say now about the node and module of a model key, from their want, have and stuck key.
|
||||
func (m *Model) motionOf(hk string) []MotionEvent {
|
||||
w, ok := m.wants[hk]
|
||||
if !ok || w.err != "" || w.value.Removed || w.value.FP == "" {
|
||||
return nil
|
||||
}
|
||||
node, module := splitKey(hk)
|
||||
h, has := m.haves[hk]
|
||||
has = has && h.err == ""
|
||||
body := MotionBody{Node: node, Module: module, ToFP: w.value.FP, Composed: w.value.Composed, Since: w.at, BoundSeconds: MotionBoundSeconds}
|
||||
body.At = w.at
|
||||
if has {
|
||||
body.FromFP = h.have.FP
|
||||
if h.at.After(body.At) {
|
||||
body.At = h.at
|
||||
}
|
||||
}
|
||||
id := func(kind string) string {
|
||||
return kind + "." + node + "." + module + "." + strings.TrimPrefix(w.value.Composed, "sha256:")
|
||||
}
|
||||
applied := has && h.have.FP == w.value.FP && (h.have.Composed == "" || h.have.Composed == w.value.Composed)
|
||||
var out []MotionEvent
|
||||
say := func(kind string, b MotionBody) {
|
||||
if m.told[id(kind)] {
|
||||
return
|
||||
}
|
||||
m.told[id(kind)] = true
|
||||
out = append(out, MotionEvent{Type: kind, ID: id(kind), Body: b})
|
||||
}
|
||||
switch {
|
||||
case applied && h.have.Phase == delivery.PhaseRunning && h.have.Health == delivery.HealthOK:
|
||||
if m.told[id(MotionMoving)] {
|
||||
b := body
|
||||
b.Outcome = "running"
|
||||
say(MotionSettled, b)
|
||||
}
|
||||
return out
|
||||
case !m.told[id(MotionMoving)]:
|
||||
say(MotionMoving, body)
|
||||
}
|
||||
stop := body
|
||||
switch {
|
||||
case has && h.have.Phase == delivery.PhaseHeld:
|
||||
stop.Outcome, stop.Words = "stopped", h.have.Detail
|
||||
case has && (h.have.Health == delivery.HealthFailed || h.have.Phase == delivery.PhaseFailed) && h.have.FP == w.value.FP:
|
||||
stop.Outcome, stop.Words = "failed", h.have.Detail
|
||||
default:
|
||||
if s, ok := m.stuckWant(w.value, node, module); ok {
|
||||
stop.Outcome, stop.Words = "stopped", s
|
||||
} else {
|
||||
return out
|
||||
}
|
||||
}
|
||||
say(MotionStuck, stop)
|
||||
return out
|
||||
}
|
||||
|
||||
// stuckWant is the watchman's words for the want of module on node, when it says the want stopped moving and not yet moved.
|
||||
func (m *Model) stuckWant(_ delivery.Want, node, module string) (string, bool) {
|
||||
wk, err := delivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
sk, err := delivery.StuckKey(delivery.AssignerWantsBucket, wk)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if st, ok := m.stuck[sk]; ok && st.err == "" && !st.value.Moved {
|
||||
return st.value.Waited, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
fpOld = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
fpNew = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
func wantE(t *testing.T, node, module string, w delivery.Want) Entry {
|
||||
t.Helper()
|
||||
key, err := delivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !w.Removed {
|
||||
w.Composed = "sha256:" + strings.Repeat("c", 64)
|
||||
}
|
||||
v, err := delivery.EncodeWant(w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AssignerWantsBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func haveFP(t *testing.T, node, module, fp string, health delivery.Health) Entry {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: health})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.HaveBucket, Key: key, Value: b, At: t0}
|
||||
}
|
||||
|
||||
func moduleE(t *testing.T, module, fp string) Entry {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: strings.Repeat("a", 64), Repository: "novox/" + module, Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.ModulesBucket, Key: module, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func sourceE(t *testing.T, revision uint64) Entry {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.SourceBucket, Key: delivery.SourceMainKey, Value: v, At: t0, Revision: revision}
|
||||
}
|
||||
|
||||
// 1906/D7 AC7: every want is a row beside the have of its node, with where it stands.
|
||||
func TestEveryWantIsARowAgainstItsHave(t *testing.T) {
|
||||
m := New()
|
||||
fill(m,
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK),
|
||||
wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed),
|
||||
wantE(t, "g14", "mail", delivery.Want{FP: fpOld, Newest: fpNew, Rule: delivery.RuleCanary, Words: "the canary ace does not run it healthy yet"}),
|
||||
wantE(t, "g14", "old", delivery.Want{Removed: true}),
|
||||
wantE(t, "novox", "web", delivery.Want{FP: fpNew}),
|
||||
)
|
||||
got := map[string]WantRow{}
|
||||
for _, r := range m.Snapshot().Wants {
|
||||
got[r.Node+"."+r.Module] = r
|
||||
}
|
||||
for key, state := range map[string]string{
|
||||
"ace.web": WantApplied, "ace.mail": WantPending, "g14.web": WantFailed, "g14.mail": WantHeldBack, "g14.old": WantRemoved, "novox.web": WantPending,
|
||||
} {
|
||||
if got[key].State != state {
|
||||
t.Errorf("%s = %+v, want state %s", key, got[key], state)
|
||||
}
|
||||
}
|
||||
if r := got["g14.mail"]; r.Rule != delivery.RuleCanary || r.Newest != fpNew || r.Words == "" {
|
||||
t.Errorf("the held-back row keeps its rule and words: %+v", r)
|
||||
}
|
||||
if r := got["ace.mail"]; r.Want != fpNew || r.Have != fpOld {
|
||||
t.Errorf("a pending row shows both: %+v", r)
|
||||
}
|
||||
// a want taken away leaves the list
|
||||
fill(m, Entry{Bucket: delivery.AssignerWantsBucket, Key: "want.novox.web", Deleted: true})
|
||||
if n := len(m.Snapshot().Wants); n != 5 {
|
||||
t.Errorf("wants = %d after a delete, want 5", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D7 AC7: a source revision is delivered when every want equals its module's fingerprint and every have equals its
|
||||
// want, healthy; one failed have shows it not delivered and names the module.
|
||||
func TestASourceRevisionIsDerivedFromWantsAndHaves(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), moduleE(t, "mail", fpNew),
|
||||
assign("ace", "web", "b-1", ""), assign("g14", "web", "b-1", ""),
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthOK),
|
||||
wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), haveFP(t, "ace", "mail", fpNew, delivery.HealthOK))
|
||||
s := m.Snapshot().Source
|
||||
if s == nil || s.Revision != 14355 || s.State != SourceDelivered || len(s.Waiting) != 0 {
|
||||
t.Fatalf("source = %+v, want revision 14355 delivered", s)
|
||||
}
|
||||
fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthFailed))
|
||||
s = m.Snapshot().Source
|
||||
if s.State != SourceNotDelivered || len(s.Waiting) != 1 || !strings.Contains(s.Waiting[0], "web") || !strings.Contains(s.Waiting[0], "g14") {
|
||||
t.Errorf("a failed have: %+v", s)
|
||||
}
|
||||
fill(m, haveFP(t, "g14", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "mail", delivery.Want{FP: fpOld}))
|
||||
s = m.Snapshot().Source
|
||||
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "mail") {
|
||||
t.Errorf("a want that is not the module's fingerprint: %+v", s)
|
||||
}
|
||||
fill(m, wantE(t, "ace", "mail", delivery.Want{FP: fpNew}), assign("novox", "web", "b-1", ""))
|
||||
s = m.Snapshot().Source
|
||||
if s.State != SourceNotDelivered || !strings.Contains(strings.Join(s.Waiting, ";"), "novox") {
|
||||
t.Errorf("a node assigned the module with no want: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoSourceRevisionHeldMeansNoStateIsDerived(t *testing.T) {
|
||||
if s := New().Snapshot().Source; s != nil {
|
||||
t.Errorf("source = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWantsPageSaysWantsAgainstHaves(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
text := wantText(m.Snapshot())
|
||||
if len(text) != 1 || text[0][0] != "ace" || text[0][1] != "web" || text[0][2] != fpNew || text[0][3] != fpOld || text[0][4] != WantPending || text[0][5] != "none" {
|
||||
t.Errorf("text = %v", text)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D7 (ADR 0539 §14): the controller's judgement of the move a want names is shown beside it, with its words.
|
||||
func TestTheJudgementOfTheMoveIsShownBesideTheWant(t *testing.T) {
|
||||
m := New()
|
||||
key, err := delivery.ComposedKey("ace", "web", fpNew)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedCannot, Words: "the composition it runs is gone"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), Entry{Bucket: delivery.ControllerComposedBucket, Key: key, Value: v, At: t0})
|
||||
rows := m.Snapshot().Wants
|
||||
if len(rows) != 1 || rows[0].Judgement != string(delivery.JudgedCannot) || !strings.Contains(rows[0].Judged, "gone") {
|
||||
t.Fatalf("rows = %+v", rows)
|
||||
}
|
||||
text := wantText(m.Snapshot())
|
||||
if len(text[0]) != 7 || text[0][5] != "cannot-judge" {
|
||||
t.Errorf("text = %v", text)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D9 (asked by the lead for issue 1946): the view says, once for each source revision, that it is delivered: the event the
|
||||
// issue tracker derives "fix delivered" from, in place of the gate's states that go in release 6 (ADR 0539 §17).
|
||||
func TestTheViewSaysOnceThatASourceRevisionIsDelivered(t *testing.T) {
|
||||
m := New()
|
||||
var said []DeliveredEvent
|
||||
m.SetDeliveredEvent(func(e DeliveredEvent) { said = append(said, e) })
|
||||
fill(m, sourceE(t, 14355), moduleE(t, "web", fpNew), assign("ace", "web", "b-1", ""),
|
||||
wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said %v before any have", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed))
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said %v for a failed have", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if len(said) != 1 || said[0].Revision != 14355 || said[0].Group != "g" || said[0].Heads["novox/web"] == "" {
|
||||
t.Fatalf("said = %+v, want revision 14355 once with its group and heads", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if len(said) != 1 {
|
||||
t.Errorf("said again for the same revision: %v", said)
|
||||
}
|
||||
// a newer revision is delivered when its wants and haves are; a revision that goes bad and good again is not said twice
|
||||
fill(m, sourceE(t, 14356))
|
||||
if len(said) != 2 || said[1].Revision != 14356 {
|
||||
t.Errorf("the next revision: %v", said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthFailed), haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if len(said) != 2 {
|
||||
t.Errorf("said again after a have failed and recovered: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (view notes): the controller no longer writes the build of an assignment (C6), so a row reads the want and the
|
||||
// have: the row shows the fingerprint wanted and waits while the node runs another.
|
||||
func TestARowReadsTheWantWhenTheAssignmentNamesNoBuild(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, assign("ace", "web", "", ""), wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
r := find(t, m.Snapshot(), "ace", "web")
|
||||
if r.Assigned != fpNew || r.Have != fpOld || !strings.Contains(r.Waits, "has not applied") {
|
||||
t.Fatalf("row = %+v, want it to wait for the want's fingerprint", r)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if r := find(t, m.Snapshot(), "ace", "web"); r.Waits != "" || r.Have != fpNew {
|
||||
t.Errorf("row = %+v, want no wait once the have names the want's fingerprint", r)
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1 (view notes, issue 1150): a module whose name holds a dot has its have under the escaped key; the row finds it,
|
||||
// and a have nothing is assigned to keeps the module's name.
|
||||
func TestADottedModuleNameIsFoundAndKept(t *testing.T) {
|
||||
m := New()
|
||||
fill(m, assign("ace", "novox.be", "", ""), wantE(t, "ace", "novox.be", delivery.Want{FP: fpNew}), haveFP(t, "ace", "novox.be", fpNew, delivery.HealthOK),
|
||||
haveFP(t, "ace", "old.name", fpNew, delivery.HealthOK))
|
||||
s := m.Snapshot()
|
||||
if r := find(t, s, "ace", "novox.be"); r.Have != fpNew || r.Waits != "" {
|
||||
t.Errorf("the dotted module's row = %+v", r)
|
||||
}
|
||||
if r := find(t, s, "ace", "old.name"); r.Status != StatusLeftBehind {
|
||||
t.Errorf("the left-behind dotted module = %+v", r)
|
||||
}
|
||||
if rows := s.Wants; len(rows) != 1 || rows[0].Module != "novox.be" || rows[0].State != WantApplied {
|
||||
t.Errorf("wants = %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D10 (asked by the lead for the messenger, ADR 0308): with no plans to say that a module is in motion, the view says it: `moving`
|
||||
// when a module's want on a node differs from its have, `settled` when the have reaches the want running and healthy, `stuck` when it
|
||||
// is held or failed or the watchman says it stopped moving; each once for a node, a module and a composition.
|
||||
func motionLog(m *Model) *[]MotionEvent {
|
||||
var said []MotionEvent
|
||||
m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) })
|
||||
for _, b := range []string{delivery.AssignerWantsBucket, delivery.HaveBucket, delivery.StuckBucket} {
|
||||
m.Initial(b)
|
||||
}
|
||||
return &said
|
||||
}
|
||||
|
||||
func types(said []MotionEvent) string {
|
||||
var out []string
|
||||
for _, e := range said {
|
||||
out = append(out, e.Type)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func TestAModuleInMotionIsSaidMovingThenSettledOnceEach(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
if len(*said) != 0 {
|
||||
t.Fatalf("said %v for a have with no want", types(*said))
|
||||
}
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if types(*said) != "moving" {
|
||||
t.Fatalf("said %q, want moving", types(*said))
|
||||
}
|
||||
mv := (*said)[0]
|
||||
if mv.Body.Node != "ace" || mv.Body.Module != "web" || mv.Body.FromFP != fpOld || mv.Body.ToFP != fpNew || mv.Body.Composed != "sha256:"+strings.Repeat("c", 64) ||
|
||||
mv.Body.Since.IsZero() || mv.Body.At.IsZero() || mv.Body.BoundSeconds != 1800 || mv.ID != "moving.ace.web."+strings.Repeat("c", 64) {
|
||||
t.Errorf("moving = %+v", mv)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpOld, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if types(*said) != "moving" {
|
||||
t.Errorf("said %q after the same state again", types(*said))
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if types(*said) != "moving,settled" || (*said)[1].Body.Outcome != "running" {
|
||||
t.Errorf("said %q, want settled with its outcome", types(*said))
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK))
|
||||
if types(*said) != "moving,settled" {
|
||||
t.Errorf("settled said twice: %q", types(*said))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleHeldOrFailedIsSaidStuckWithItsWords(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
b, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseHeld, Health: delivery.HealthWaits, Detail: "no space left"})
|
||||
fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0})
|
||||
if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" || (*said)[1].Body.Words != "no space left" {
|
||||
t.Fatalf("said %v: %+v", types(*said), *said)
|
||||
}
|
||||
fill(m, Entry{Bucket: delivery.HaveBucket, Key: "ace.web", Value: b, At: t0})
|
||||
if types(*said) != "moving,stuck" {
|
||||
t.Errorf("stuck said twice: %q", types(*said))
|
||||
}
|
||||
// failed on the wanted fingerprint
|
||||
m2 := New()
|
||||
said2 := motionLog(m2)
|
||||
fill(m2, wantE(t, "g14", "web", delivery.Want{FP: fpNew}), haveFP(t, "g14", "web", fpNew, delivery.HealthFailed))
|
||||
if types(*said2) != "moving,stuck" || (*said2)[1].Body.Outcome != "failed" {
|
||||
t.Errorf("failed: said %q", types(*said2))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWantTheWatchmanSaysStoppedMovingIsSaidStuckPastItsBound(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), haveFP(t, "ace", "web", fpOld, delivery.HealthOK))
|
||||
sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, "want.ace.web")
|
||||
v, err := delivery.EncodeStuck(delivery.Stuck{Bucket: delivery.AssignerWantsBucket, Key: "want.ace.web", Waited: "the node agent of ace to run web", Since: t0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fill(m, Entry{Bucket: delivery.StuckBucket, Key: sk, Value: v, At: t0})
|
||||
if types(*said) != "moving,stuck" || (*said)[1].Body.Outcome != "stopped" {
|
||||
t.Errorf("said %q: %+v", types(*said), *said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatSettlesWithoutBeingSeenMovingSaysNothing(t *testing.T) {
|
||||
m := New()
|
||||
said := motionLog(m)
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), wantE(t, "ace", "web", delivery.Want{FP: fpNew}))
|
||||
if len(*said) != 0 {
|
||||
t.Errorf("said %v for a module that never moved", types(*said))
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing is said of a module in motion before the buckets have been read: a want read ahead of its have is not a node behind. What is
|
||||
// in motion when they have been read is said then, and a module that settles after is said settled.
|
||||
func TestMotionIsSaidOnlyOnceTheBucketsHaveBeenRead(t *testing.T) {
|
||||
m := New()
|
||||
var said []MotionEvent
|
||||
m.SetMotionEvent(func(e MotionEvent) { said = append(said, e) })
|
||||
fill(m, wantE(t, "ace", "web", delivery.Want{FP: fpNew}), wantE(t, "ace", "mail", delivery.Want{FP: fpNew}))
|
||||
m.Initial(delivery.AssignerWantsBucket)
|
||||
fill(m, haveFP(t, "ace", "web", fpNew, delivery.HealthOK), haveFP(t, "ace", "mail", fpOld, delivery.HealthOK))
|
||||
m.Initial(delivery.HaveBucket)
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("said %v before the stuck keys were read", types(said))
|
||||
}
|
||||
m.Initial(delivery.StuckBucket)
|
||||
if len(said) != 1 || said[0].Type != MotionMoving || said[0].Body.Module != "mail" {
|
||||
t.Fatalf("said %v: %+v, want only mail moving (web is already there)", types(said), said)
|
||||
}
|
||||
fill(m, haveFP(t, "ace", "mail", fpNew, delivery.HealthOK))
|
||||
if types(said) != "moving,settled" {
|
||||
t.Errorf("said %q", types(said))
|
||||
}
|
||||
}
|
||||
@@ -50,6 +50,7 @@ type manifest struct {
|
||||
Reads []string `json:"reads"`
|
||||
Uses []string `json:"uses"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
@@ -83,17 +84,21 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
if m.Module != "mesh-delivery-view" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("reads = %v, want exactly %v", got, want)
|
||||
}
|
||||
for _, k := range []string{"emits", "consumes", "provides", "claims"} {
|
||||
for _, k := range []string{"consumes", "provides", "claims"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the view writes only its answers", k)
|
||||
}
|
||||
}
|
||||
// The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves.
|
||||
if !reflect.DeepEqual(m.Emits, []string{"delivered", "moving", "settled", "stuck"}) {
|
||||
t.Errorf("emits = %v, want exactly delivered, moving, settled and stuck (1906/D9, D10)", m.Emits)
|
||||
}
|
||||
// The view uses no build seat: it reads their workers by the broker's named grant and publishes no ask (ADR 0525 §2).
|
||||
if len(m.Uses) != 0 {
|
||||
t.Errorf("uses = %v, want none", m.Uses)
|
||||
@@ -110,7 +115,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
}
|
||||
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
||||
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
if want := []string{"indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves"}; !reflect.DeepEqual(m.Tools, want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -9,7 +9,17 @@
|
||||
"indexer_candidate",
|
||||
"indexer_candidates",
|
||||
"builder_build",
|
||||
"builder_builds"
|
||||
"builder_builds",
|
||||
"assigner_wants",
|
||||
"assigner_want",
|
||||
"assigner_differences",
|
||||
"node_haves"
|
||||
],
|
||||
"emits": [
|
||||
"delivered",
|
||||
"moving",
|
||||
"settled",
|
||||
"stuck"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "answers", "history": 5 }
|
||||
@@ -17,10 +27,13 @@
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
"mesh-controller.builds",
|
||||
"mesh-controller.composed",
|
||||
"mesh-delivery-assigner.asks",
|
||||
"mesh-delivery-assigner.differences",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-gate.asks",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -99,6 +100,8 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
|
||||
{Bucket: delivery.StuckBucket, History: 20},
|
||||
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
|
||||
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
|
||||
{Bucket: delivery.AssignerWantsBucket, History: 20},
|
||||
{Bucket: delivery.HaveBucket, History: 20},
|
||||
} {
|
||||
kv, err := js.CreateKeyValue(ctx, c)
|
||||
if err != nil {
|
||||
@@ -265,6 +268,8 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
|
||||
{Bucket: delivery.StuckBucket, History: 20},
|
||||
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
|
||||
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
|
||||
{Bucket: delivery.AssignerWantsBucket, History: 20},
|
||||
{Bucket: delivery.HaveBucket, History: 20},
|
||||
} {
|
||||
kv, err := js.CreateKeyValue(ctx, c)
|
||||
if err != nil {
|
||||
@@ -302,3 +307,134 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
|
||||
}
|
||||
t.Fatalf("no pulse for the ask running before the start (%s)", pk)
|
||||
}
|
||||
|
||||
// TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt (1906/D6, ADR 0539 §19): started whole, the watchman mirrors a want of
|
||||
// the assigner as a pulse owed by the node's agent, one that stood before its start as well, and the node-engine's have that
|
||||
// names the want's fingerprint deletes the pulse and writes no stuck key.
|
||||
func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "w")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
srv, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ctl.Close)
|
||||
js, err := jetstream.New(ctl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
kvs := map[string]jetstream.KeyValue{}
|
||||
for _, c := range []jetstream.KeyValueConfig{
|
||||
{Bucket: delivery.PresenceBucket, History: 20},
|
||||
{Bucket: delivery.StuckBucket, History: 20},
|
||||
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
|
||||
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
|
||||
{Bucket: delivery.AssignerWantsBucket, History: 20},
|
||||
{Bucket: delivery.HaveBucket, History: 20},
|
||||
} {
|
||||
kv, err := js.CreateKeyValue(ctx, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[c.Bucket] = kv
|
||||
}
|
||||
fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
putWant := func(node, module, fp string) {
|
||||
t.Helper()
|
||||
body, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, key, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
putHave := func(node, module, fp string) {
|
||||
t.Helper()
|
||||
body, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveKey(node, module)
|
||||
if _, err := kvs[delivery.HaveBucket].Put(ctx, key, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
putHave("n2", "web", fpOld)
|
||||
putWant("n2", "web", fpNew) // stood before the start
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-watchman", User: "n1.mesh-delivery-watchman", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
pulse := func(node, module string) string {
|
||||
wk, _ := delivery.WantKey(node, module)
|
||||
pk, err := delivery.PulseKey(delivery.AssignerWantsBucket, wk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return pk
|
||||
}
|
||||
until := func(what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("never saw: %s", what)
|
||||
}
|
||||
has := func(key string) bool { _, err := kvs[delivery.PulseBucket].Get(ctx, key); return err == nil }
|
||||
until("a pulse for the want that stood before the start", func() bool { return has(pulse("n2", "web")) })
|
||||
putWant("n2", "mail", fpNew)
|
||||
until("a pulse for a want written after the start", func() bool { return has(pulse("n2", "mail")) })
|
||||
e, err := kvs[delivery.PulseBucket].Get(ctx, pulse("n2", "web"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, err := delivery.DecodePulse(e.Value()); err != nil || !strings.Contains(p.Owes, "n2") {
|
||||
t.Errorf("the pulse = %+v, %v; want it owed by the agent of n2", p, err)
|
||||
}
|
||||
putHave("n2", "web", fpNew)
|
||||
until("the pulse of the want its have names deleted", func() bool { return !has(pulse("n2", "web")) })
|
||||
if !has(pulse("n2", "mail")) {
|
||||
t.Errorf("the pulse of another want ended with web's have")
|
||||
}
|
||||
for _, m := range []string{"web"} {
|
||||
wk, _ := delivery.WantKey("n2", m)
|
||||
sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, wk)
|
||||
if _, err := kvs[delivery.StuckBucket].Get(ctx, sk); err == nil {
|
||||
t.Errorf("a stuck key was written for the want of %s that was applied", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,11 +229,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
var kvs struct{ presence, pulse, stuck jetstream.KeyValue }
|
||||
var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue }
|
||||
for _, b := range []struct {
|
||||
name string
|
||||
into *jetstream.KeyValue
|
||||
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} {
|
||||
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck},
|
||||
{delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} {
|
||||
if *b.into, err = bucket(b.name); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -280,6 +281,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent
|
||||
// until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse.
|
||||
haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
|
||||
// keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same.
|
||||
taken := map[string]jetstream.KeyWatcher{}
|
||||
@@ -337,6 +348,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
go func() {
|
||||
errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) })
|
||||
}()
|
||||
go func() {
|
||||
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
|
||||
}()
|
||||
go func() {
|
||||
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
|
||||
}()
|
||||
for lang, sw := range taken {
|
||||
lang, sw := lang, sw
|
||||
go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }()
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -52,3 +55,96 @@ func (w *Watchman) Step(language, key string, value []byte, op Op) {
|
||||
}
|
||||
w.mirrorWithin(w.stepBound, bucket, id, describe("the ask %s to end on %s", id, node), node, since, 0)
|
||||
}
|
||||
|
||||
// A want is a move too (novox/hq ADR 0539 §19, issue 1906): the assigner writes `want.<node>.<module>` in its wants bucket,
|
||||
// and the node's agent owes the node-engine's have naming the want's fingerprint. The pulse is the pulse of that bucket under
|
||||
// the want's key, owed by the node's agent, and it begins when the assigner wrote the want. The have naming the fingerprint
|
||||
// ends it; a want that is removed, or a key deleted, ends it as well. Nothing renews it: the bound is how long a node may
|
||||
// take to apply a want, the same as a build step's (StepBound). Keys of that bucket that are not a want of a module (a hold,
|
||||
// and the node's own key) are left.
|
||||
|
||||
// Want is a change of a key of the assigner's wants bucket, read from the watch (at the start the keys that stand, then each
|
||||
// change), at the time the entry was written.
|
||||
func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
|
||||
node, module, err := delivery.ParseWantKey(key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch op {
|
||||
case Expired:
|
||||
return
|
||||
case Delete:
|
||||
delete(w.wants, key)
|
||||
w.moved(delivery.AssignerWantsBucket, key)
|
||||
return
|
||||
}
|
||||
want, err := delivery.DecodeWant(value)
|
||||
if err != nil {
|
||||
w.say("the want %s cannot be read, and is left: %v", key, err)
|
||||
return
|
||||
}
|
||||
if want.Removed {
|
||||
delete(w.wants, key)
|
||||
w.moved(delivery.AssignerWantsBucket, key)
|
||||
return
|
||||
}
|
||||
w.wants[key] = want.FP
|
||||
if w.have(node, module) == want.FP {
|
||||
w.moved(delivery.AssignerWantsBucket, key)
|
||||
return
|
||||
}
|
||||
if at.IsZero() {
|
||||
at = w.now()
|
||||
}
|
||||
w.mirrorWithin(w.stepBound, delivery.AssignerWantsBucket, key,
|
||||
describe("the node agent of %s to run %s at %s", node, module, shortFP(want.FP)), want.FP, at, 0)
|
||||
}
|
||||
|
||||
// Have is a change of a key of the node-engine's have bucket: the fingerprint a node runs of a module. One that names the
|
||||
// fingerprint of the node's want for the module ends the want's pulse. A node's own key and a key deleted are left.
|
||||
func (w *Watchman) Have(key string, value []byte, op Op) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
switch op {
|
||||
case Expired:
|
||||
return
|
||||
case Delete:
|
||||
delete(w.haves, key)
|
||||
return
|
||||
}
|
||||
have, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
w.say("the have %s cannot be read, and is left: %v", key, err)
|
||||
return
|
||||
}
|
||||
w.haves[key] = have.FP
|
||||
wk, err := delivery.WantKey(node, module)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if fp, ok := w.wants[wk]; ok && fp == have.FP {
|
||||
w.moved(delivery.AssignerWantsBucket, wk)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watchman) have(node, module string) string {
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return w.haves[key]
|
||||
}
|
||||
|
||||
func shortFP(fp string) string {
|
||||
hex := strings.TrimPrefix(fp, delivery.SourcePrefix2)
|
||||
if len(hex) > 12 {
|
||||
hex = hex[:12]
|
||||
}
|
||||
return hex
|
||||
}
|
||||
|
||||
@@ -57,6 +57,8 @@ type Watchman struct {
|
||||
presence map[string]presence
|
||||
pulses map[string]*pulse // live pulses, by pulse key
|
||||
stuck map[string]delivery.Stuck
|
||||
wants map[string]string // the fingerprint of each want that stands, by want key
|
||||
haves map[string]string // the fingerprint each node runs of a module, by have key
|
||||
bound time.Duration
|
||||
// stepBound is how long a build step may run once a node took the ask; a test shortens it.
|
||||
stepBound time.Duration
|
||||
@@ -65,7 +67,7 @@ type Watchman struct {
|
||||
// New is a watchman writing to out, reading the time from now.
|
||||
func New(out Out, now func() time.Time) *Watchman {
|
||||
return &Watchman{out: out, now: now, counts: map[string]int{}, presence: map[string]presence{},
|
||||
pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, bound: delivery.PulseMoveBound, stepBound: StepBound}
|
||||
pulses: map[string]*pulse{}, stuck: map[string]delivery.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, bound: delivery.PulseMoveBound, stepBound: StepBound}
|
||||
}
|
||||
|
||||
// Logf sets where a write that failed is said; it is silent until set.
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -531,3 +532,91 @@ func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908
|
||||
t.Errorf("stuck %v pulses %v", o2.stuck, o2.pulses)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D6 (ADR 0539 §19): a want is a pulse owed by the node's agent from when the assigner wrote it, until the have names
|
||||
// its fingerprint.
|
||||
func wantValue(t *testing.T, fp string, removed bool) []byte {
|
||||
t.Helper()
|
||||
w := delivery.Want{FP: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed}
|
||||
if removed {
|
||||
w.Composed = ""
|
||||
}
|
||||
b, err := delivery.EncodeWant(w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func haveValue(t *testing.T, fp string) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
var (
|
||||
fpOld = delivery.SourcePrefix2 + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
fpNew = delivery.SourcePrefix2 + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
)
|
||||
|
||||
func TestAWantIsAPulseOwedByTheNodeAndTheHaveNamingItEndsIt(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
c.t = t0.Add(10 * time.Second)
|
||||
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")
|
||||
p, ok := o.pulses[pk]
|
||||
if !ok || p.Key != "want.ace.web" || !strings.Contains(p.Owes, "ace") || !strings.Contains(p.Owes, "web") {
|
||||
t.Fatalf("pulses = %v", o.pulses)
|
||||
}
|
||||
w.Have("ace.web", haveValue(t, fpOld), Put) // the node still runs the old one
|
||||
if _, ok := o.pulses[pk]; !ok {
|
||||
t.Fatalf("a have of another fingerprint ended the pulse")
|
||||
}
|
||||
w.Have("ace.web", haveValue(t, fpNew), Put)
|
||||
if _, ok := o.pulses[pk]; ok {
|
||||
t.Errorf("the have naming the want's fingerprint left the pulse standing: %v", o.pulses)
|
||||
}
|
||||
if len(o.stuck) != 0 {
|
||||
t.Errorf("stuck = %v", o.stuck)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHaveThatCameBeforeTheWantEndsItAtOnceAndARemovedWantHasNoPulse(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
c.t = t0.Add(time.Second)
|
||||
w.Have("ace.web", haveValue(t, fpNew), Put)
|
||||
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
w.Want("want.ace.mail", wantValue(t, "", true), Put, t0)
|
||||
if len(o.pulses) != 0 {
|
||||
t.Errorf("pulses = %v", o.pulses)
|
||||
}
|
||||
// the want moving on to a new fingerprint makes a new pulse; the key `want.ace.node` and a held key are left
|
||||
w.Want("want.ace.web", wantValue(t, fpOld, false), Put, t0.Add(time.Second))
|
||||
w.Want("want.ace.node", []byte(`{}`), Put, t0)
|
||||
w.Want("held.web.bbbb", []byte(`{}`), Put, t0)
|
||||
if len(o.pulses) != 1 {
|
||||
t.Errorf("pulses = %v, want the web want's only", o.pulses)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) {
|
||||
o, c := newOut(), &clock{t: t0}
|
||||
w := New(o, c.now)
|
||||
w.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
pk := pulseKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")
|
||||
c.t = t0.Add(31 * time.Minute)
|
||||
w.Pulse(pk, Expired, c.t)
|
||||
if s, ok := o.stuck[stuckKeyOf(t, delivery.AssignerWantsBucket, "want.ace.web")]; !ok || s.Moved || !s.Since.Equal(t0) {
|
||||
t.Fatalf("stuck = %v", o.stuck)
|
||||
}
|
||||
w2, o2, c2 := setup()
|
||||
c2.t = t0.Add(time.Second)
|
||||
w2.Want("want.ace.web", wantValue(t, fpNew, false), Put, t0)
|
||||
w2.Want("want.ace.web", nil, Delete, t0)
|
||||
if len(o2.pulses) != 0 {
|
||||
t.Errorf("a deleted want left its pulse: %v", o2.pulses)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps"}) {
|
||||
if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-delivery-assigner.wants", "node-engine.have"}) {
|
||||
t.Errorf("reads = %v", reads)
|
||||
}
|
||||
for _, k := range []string{"emits", "context", "items", "consumes", "provides", "claims"} {
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-image.steps",
|
||||
"mesh-build-node.steps"
|
||||
"mesh-build-node.steps",
|
||||
"mesh-delivery-assigner.wants",
|
||||
"node-engine.have"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "presence", "history": 20 },
|
||||
|
||||
Reference in New Issue
Block a user