novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

The watchman mirrors each want of the assigner as a pulse owed by the node's agent until the have names its fingerprint (issue 1906 D6)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 16:22:02 +02:00
parent 024a3787da
commit 9d2e9dc6c9
7 changed files with 344 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go
+136
View File
@@ -9,6 +9,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -99,6 +100,8 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
{Bucket: delivery.StuckBucket, History: 20},
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
{Bucket: delivery.AssignerWantsBucket, History: 20},
{Bucket: delivery.HaveBucket, History: 20},
} {
kv, err := js.CreateKeyValue(ctx, c)
if err != nil {
@@ -265,6 +268,8 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
{Bucket: delivery.StuckBucket, History: 20},
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
{Bucket: delivery.AssignerWantsBucket, History: 20},
{Bucket: delivery.HaveBucket, History: 20},
} {
kv, err := js.CreateKeyValue(ctx, c)
if err != nil {
@@ -302,3 +307,134 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
}
t.Fatalf("no pulse for the ask running before the start (%s)", pk)
}
// TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt (1906/D6, ADR 0539 §19): started whole, the watchman mirrors a want of
// the assigner as a pulse owed by the node's agent, one that stood before its start as well, and the node-engine's have that
// names the want's fingerprint deletes the pulse and writes no stuck key.
func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "w")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
conf := filepath.Join(dir, "bus.conf")
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatal(err)
}
opts.NoLog, opts.NoSigs = true, true
srv, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(ctl.Close)
js, err := jetstream.New(ctl)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
kvs := map[string]jetstream.KeyValue{}
for _, c := range []jetstream.KeyValueConfig{
{Bucket: delivery.PresenceBucket, History: 20},
{Bucket: delivery.StuckBucket, History: 20},
{Bucket: delivery.PulseBucket, LimitMarkerTTL: time.Second},
{Bucket: delivery.StepsBucket("mesh-build-go"), History: 5},
{Bucket: delivery.AssignerWantsBucket, History: 20},
{Bucket: delivery.HaveBucket, History: 20},
} {
kv, err := js.CreateKeyValue(ctx, c)
if err != nil {
t.Fatal(err)
}
kvs[c.Bucket] = kv
}
fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
putWant := func(node, module, fp string) {
t.Helper()
body, err := delivery.EncodeWant(delivery.Want{FP: fp})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.WantKey(node, module)
if _, err := kvs[delivery.AssignerWantsBucket].Put(ctx, key, body); err != nil {
t.Fatal(err)
}
}
putHave := func(node, module, fp string) {
t.Helper()
body, err := delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveKey(node, module)
if _, err := kvs[delivery.HaveBucket].Put(ctx, key, body); err != nil {
t.Fatal(err)
}
}
putHave("n2", "web", fpOld)
putWant("n2", "web", fpNew) // stood before the start
broker := filepath.Join(dir, "broker")
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-watchman", User: "n1.mesh-delivery-watchman", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
pulse := func(node, module string) string {
wk, _ := delivery.WantKey(node, module)
pk, err := delivery.PulseKey(delivery.AssignerWantsBucket, wk)
if err != nil {
t.Fatal(err)
}
return pk
}
until := func(what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if ok() {
return
}
}
t.Fatalf("never saw: %s", what)
}
has := func(key string) bool { _, err := kvs[delivery.PulseBucket].Get(ctx, key); return err == nil }
until("a pulse for the want that stood before the start", func() bool { return has(pulse("n2", "web")) })
putWant("n2", "mail", fpNew)
until("a pulse for a want written after the start", func() bool { return has(pulse("n2", "mail")) })
e, err := kvs[delivery.PulseBucket].Get(ctx, pulse("n2", "web"))
if err != nil {
t.Fatal(err)
}
if p, err := delivery.DecodePulse(e.Value()); err != nil || !strings.Contains(p.Owes, "n2") {
t.Errorf("the pulse = %+v, %v; want it owed by the agent of n2", p, err)
}
putHave("n2", "web", fpNew)
until("the pulse of the want its have names deleted", func() bool { return !has(pulse("n2", "web")) })
if !has(pulse("n2", "mail")) {
t.Errorf("the pulse of another want ended with web's have")
}
for _, m := range []string{"web"} {
wk, _ := delivery.WantKey("n2", m)
sk, _ := delivery.StuckKey(delivery.AssignerWantsBucket, wk)
if _, err := kvs[delivery.StuckBucket].Get(ctx, sk); err == nil {
t.Errorf("a stuck key was written for the want of %s that was applied", m)
}
}
}
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go
+19 -2
View File
@@ -229,11 +229,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
return kv, nil
}
var kvs struct{ presence, pulse, stuck jetstream.KeyValue }
var kvs struct{ presence, pulse, stuck, wants, haves jetstream.KeyValue }
for _, b := range []struct {
name string
into *jetstream.KeyValue
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck}} {
}{{delivery.PresenceBucket, &kvs.presence}, {delivery.PulseBucket, &kvs.pulse}, {delivery.StuckBucket, &kvs.stuck},
{delivery.AssignerWantsBucket, &kvs.wants}, {delivery.HaveBucket, &kvs.haves}} {
if *b.into, err = bucket(b.name); err != nil {
return err
}
@@ -280,6 +281,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent
// until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse.
haveSeen, err := settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
if err != nil {
return err
}
wantSeen, err := settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
if err != nil {
return err
}
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
// keeps no steps bucket (yet) is said and left; the asks of its stream are mirrored all the same.
taken := map[string]jetstream.KeyWatcher{}
@@ -337,6 +348,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go func() {
errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) })
}()
go func() {
errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) })
}()
go func() {
errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) })
}()
for lang, sw := range taken {
lang, sw := lang, sw
go func() { errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) { stepTaken(w, lang, e) }) }()