Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a5b763f8a2 | ||
|
|
50a2e4c4e2 | ||
|
|
4169faf3f0 | ||
|
|
103eec8855 | ||
|
|
8a62f23afe | ||
|
|
84395420ee | ||
|
|
9740842af7 | ||
|
|
588cfd54bb | ||
|
|
4ccd78c9ec | ||
|
|
6fe7b7dd6c |
@@ -19,7 +19,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
@@ -126,8 +125,8 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
||||
return done, stopped
|
||||
}
|
||||
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
|
||||
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
|
||||
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
|
||||
// having opened every bucket it names.
|
||||
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
done := b.start(t)
|
||||
@@ -151,21 +150,9 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||
}
|
||||
break
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != len(m.State) {
|
||||
t.Fatalf("the assigner said %+v, want the %d states its manifest declares", said, len(m.State))
|
||||
}
|
||||
for _, s := range m.State {
|
||||
if !said.Readable(s.Name) {
|
||||
t.Errorf("the assigner said %+v: its state %s is not readable", said, s.Name)
|
||||
}
|
||||
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != 4 || !said.Readable("asks") ||
|
||||
!said.Readable("wants") || !said.Readable("differences") || !said.Readable("compositions") {
|
||||
t.Fatalf("the assigner said %+v, want its four states, each readable", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,42 +169,6 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||
t.Errorf("without %s the assigner started", missing)
|
||||
}
|
||||
}
|
||||
// ADR 0546: without the puts' bucket it starts all the same, and once the bucket is made (by the controller at its start,
|
||||
// ADR 0201) a pass opens it, so a put is taken further without a restart.
|
||||
was := PutsReopenEvery
|
||||
PutsReopenEvery = 0
|
||||
defer func() { PutsReopenEvery = was }()
|
||||
b := startBed(t, delivery.AssignerPutsBucket)
|
||||
done := b.start(t)
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("without %s the assigner stopped: %v", delivery.AssignerPutsBucket, err)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
puts, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerPutsBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pkey, _ := delivery.PutKey("n1", "web")
|
||||
asked := delivery.Put{FP: delivery.SourcePrefix2 + strings.Repeat("b", 64), Why: "a try", At: time.Now()}.Said(time.Now(), delivery.PutAsked, "asked")
|
||||
raw, _ := delivery.EncodePut(asked)
|
||||
if _, err := puts.Put(b.ctx, pkey, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
|
||||
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
|
||||
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, err := puts.Get(b.ctx, pkey); err == nil {
|
||||
if got, err := delivery.DecodePut(e.Value()); err == nil && got.Step == delivery.PutRefused && strings.Contains(got.Words, "not assigned") {
|
||||
break
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the put was not taken further once its bucket was made")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// putJSON writes a value into one of the bed's buckets.
|
||||
@@ -330,12 +281,9 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1297/B4 on the bus, 1906/D5: the controller sends ace, g14 and shanks builds while the canary ace does not run F healthy yet.
|
||||
// Started whole, the assigner wants F on the canary and writes no difference at all: there is no send to compare with (ADR 0539
|
||||
// §8). 1945/AC19, AC21: shanks, which runs the older fingerprint (composed and granted) with no want, gets its first want at that
|
||||
// fingerprint, never at F; g14, which runs a build of the old form, gets none, so web is on neither list there and the assigner
|
||||
// raises the condition naming g14, web and the canary rule; once ace runs F healthy, g14's first want names F and the condition is
|
||||
// cleared.
|
||||
// 1297/B4 on the bus, 1906/D5: the controller sends ace and g14 builds while the canary ace does not run F healthy yet.
|
||||
// Started whole, the assigner wants F on the canary, holds g14 back with no want, and writes no difference at all: there
|
||||
// is no send to compare with (ADR 0539 §8).
|
||||
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||
@@ -348,21 +296,19 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
}
|
||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
for _, fp := range []string{fNew, fOld} {
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
}
|
||||
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
|
||||
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
|
||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||
b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
b.compose(t, "g14", "web", fNew, 'c', delivery.JudgedNeither)
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
||||
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
|
||||
}
|
||||
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
|
||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
|
||||
b.start(t)
|
||||
|
||||
wkey, _ := delivery.WantKey("ace", "web")
|
||||
@@ -370,41 +316,12 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if err != nil || want.FP != fNew {
|
||||
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||
}
|
||||
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
|
||||
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
|
||||
}
|
||||
var neither asks.Condition
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
|
||||
neither = c
|
||||
}
|
||||
}
|
||||
}
|
||||
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
|
||||
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
|
||||
t.Errorf("differences were written: %v", keys)
|
||||
}
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
|
||||
t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
|
||||
}
|
||||
|
||||
// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
|
||||
// there by itself: the condition is cleared.
|
||||
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
|
||||
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
|
||||
t.Errorf("g14's want = %+v, want F with its composition", w)
|
||||
}
|
||||
cleared := false
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
|
||||
c, ok := b.conditionsSaid(t)[neither.Key]
|
||||
cleared = ok && c.State == asks.StateCleared
|
||||
}
|
||||
if !cleared {
|
||||
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
|
||||
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,14 +341,12 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||
b.start(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
|
||||
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
|
||||
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
|
||||
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
|
||||
t.Fatalf("held = %+v, %v", h, err)
|
||||
}
|
||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
|
||||
if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
|
||||
t.Fatalf("the ask = %+v, %v", a, err)
|
||||
}
|
||||
}
|
||||
@@ -498,7 +413,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
||||
key := ""
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
|
||||
if c.State == asks.StateRaised && c.Subject.ID == "mail" {
|
||||
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
|
||||
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
|
||||
}
|
||||
@@ -665,15 +580,8 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
|
||||
return last
|
||||
}
|
||||
|
||||
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
|
||||
func (b *bed) answer(t *testing.T, key, option string) {
|
||||
t.Helper()
|
||||
a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.answerTo(t, key, option, a.Ask.ID)
|
||||
}
|
||||
// answer says the teller wrote the operator's answer to the ask at key.
|
||||
func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") }
|
||||
|
||||
// answerTo is answer to the ask with this id (the router's warrant names it).
|
||||
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
|
||||
@@ -717,8 +625,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
cleared := false
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
|
||||
for _, c := range b.conditionsSaid(t) {
|
||||
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared)
|
||||
}
|
||||
if cleared {
|
||||
return
|
||||
@@ -865,81 +773,3 @@ func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
||||
t.Errorf("the user-list revision was asked")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
|
||||
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
|
||||
// cleared saying why, and the want stays at the fingerprint.
|
||||
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
b.start(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||
b.get(t, delivery.AssignerWantsBucket, hkey)
|
||||
b.get(t, delivery.AssignerAsksBucket, akey)
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
|
||||
continue
|
||||
}
|
||||
for k, c := range b.conditionsSaid(t) {
|
||||
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
|
||||
t.Errorf("the want = %+v", w)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
|
||||
}
|
||||
|
||||
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
|
||||
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
|
||||
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||
b := startBed(t, "")
|
||||
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||
_, stop := b.startStoppable(t)
|
||||
hkey, _ := delivery.HeldKey("web", fNew)
|
||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||
first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the hold was not lifted")
|
||||
}
|
||||
}
|
||||
stop()
|
||||
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
|
||||
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||
b.start(t)
|
||||
again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.Ask.ID == first.Ask.ID {
|
||||
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
|
||||
}
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -30,15 +29,6 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
if os.Getenv("MESH_BROKER_FILE") == "" {
|
||||
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
|
||||
if err := serveTools(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
@@ -69,37 +59,29 @@ var readBuckets = []string{
|
||||
const goBuildModule = "mesh-build-go"
|
||||
|
||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
|
||||
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
|
||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
|
||||
|
||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||
// and the teller, its puts by whoever follows one (ADR 0546).
|
||||
// and the teller.
|
||||
var keptStates = []announce.State{
|
||||
{Name: "asks", Readable: true},
|
||||
{Name: "compositions", Readable: true},
|
||||
{Name: "differences", Readable: true},
|
||||
{Name: "puts", Readable: true},
|
||||
{Name: "wants", Readable: true},
|
||||
}
|
||||
|
||||
// buckets is every bucket the assigner opened, by name.
|
||||
type buckets map[string]jetstream.KeyValue
|
||||
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named with its grant (issue 1945),
|
||||
// since a pass without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
|
||||
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
|
||||
// without it and takes no put further until it starts again (ADR 0546).
|
||||
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
|
||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
||||
// without it would write wants from half the mesh.
|
||||
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
|
||||
out := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if name == delivery.AssignerPutsBucket && err != nil {
|
||||
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
|
||||
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
||||
}
|
||||
out[name] = kv
|
||||
}
|
||||
@@ -136,7 +118,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kvs, err := open(ctx, js, logger.Printf)
|
||||
kvs, err := open(ctx, js)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -151,14 +133,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||
}
|
||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
|
||||
}
|
||||
|
||||
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||
type emitting struct{ *process.Conn }
|
||||
|
||||
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
|
||||
|
||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||
var watched = []string{
|
||||
@@ -179,7 +156,7 @@ var watched = []string{
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
|
||||
w := newWorld()
|
||||
dirty := make(chan struct{}, 1)
|
||||
wake := func() {
|
||||
@@ -188,15 +165,13 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
default:
|
||||
}
|
||||
}
|
||||
errs := make(chan error, len(watched)+2)
|
||||
settled := make(chan struct{}, len(watched)+2)
|
||||
errs := make(chan error, len(watched)+1)
|
||||
settled := make(chan struct{}, len(watched)+1)
|
||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
|
||||
}
|
||||
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
|
||||
optional := name == delivery.AssignerPutsBucket
|
||||
go func() {
|
||||
defer kw.Stop()
|
||||
for {
|
||||
@@ -206,17 +181,11 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
return
|
||||
case e, ok := <-kw.Updates():
|
||||
if !ok {
|
||||
if optional {
|
||||
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
|
||||
return
|
||||
}
|
||||
errs <- fmt.Errorf("the watch of %s ended", name)
|
||||
return
|
||||
}
|
||||
if e == nil {
|
||||
if !optional { // the puts never gate the first pass, so never the wants (ADR 0546)
|
||||
settled <- struct{}{}
|
||||
}
|
||||
settled <- struct{}{}
|
||||
continue
|
||||
}
|
||||
var value []byte
|
||||
@@ -230,39 +199,6 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
// The puts' watch is not waited for before the first pass: a watch of them that ends before it handed what stands only
|
||||
// stops the puts, never the wants.
|
||||
if kvs[delivery.AssignerPutsBucket] != nil {
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
}
|
||||
}
|
||||
// The puts' bucket is made by the controller from the manifest's states at its start (ADR 0201), and the assigner's
|
||||
// credential reads it once the bus's user list carries the grant: neither waits for the assigner. So while it is not open,
|
||||
// a pass started by a change opens it again, at most once a minute, and from then on the puts are watched and taken
|
||||
// further, without a restart.
|
||||
var triedPuts time.Time
|
||||
reopenPuts := func(p *passer) {
|
||||
if kvs[delivery.AssignerPutsBucket] != nil || time.Since(triedPuts) < PutsReopenEvery {
|
||||
return
|
||||
}
|
||||
triedPuts = time.Now()
|
||||
kv, err := js.KeyValue(ctx, delivery.AssignerPutsBucket)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
kvs[delivery.AssignerPutsBucket] = kv
|
||||
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||
logger.Printf("%v: no put is taken further until it is opened again", err)
|
||||
delete(kvs, delivery.AssignerPutsBucket)
|
||||
return
|
||||
}
|
||||
if bus != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
|
||||
}
|
||||
for _, name := range watched {
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
@@ -286,9 +222,6 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||
}
|
||||
wake()
|
||||
for {
|
||||
select {
|
||||
@@ -298,15 +231,11 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
||||
return err
|
||||
case <-settled:
|
||||
case <-dirty:
|
||||
reopenPuts(p)
|
||||
p.pass(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
|
||||
var PutsReopenEvery = time.Minute
|
||||
|
||||
// sayVia publishes a condition on the operator-channel seat's one subject for conditions, under the assigner's own name
|
||||
// (ADR 0437 clause 9).
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) error {
|
||||
|
||||
@@ -2,23 +2,15 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||
@@ -56,53 +48,6 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||
}
|
||||
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||
for _, c := range []struct {
|
||||
missing string
|
||||
err error
|
||||
starts bool
|
||||
}{
|
||||
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
|
||||
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
|
||||
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
|
||||
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
|
||||
} {
|
||||
var logged string
|
||||
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
|
||||
switch {
|
||||
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
|
||||
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||
}
|
||||
}
|
||||
// And a watch of the puts that ends before it handed what stands (its subscription closed, its grant withdrawn) does not
|
||||
// keep the first pass from being made: the wants are written all the same.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
wrote := make(chan string, 8)
|
||||
kvs := buckets{}
|
||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||
kvs[name] = &watchKV{memKV: &memKV{values: map[string][]byte{}}}
|
||||
}
|
||||
ckey, _ := delivery.ComposedNodeKey("ace")
|
||||
cd, err := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("d", 64), Judgement: delivery.JudgedUnchanged})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
|
||||
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
|
||||
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
|
||||
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
|
||||
select {
|
||||
case key := <-wrote:
|
||||
if key != "want.ace.node" {
|
||||
t.Errorf("the first pass wrote %s", key)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Error("a puts watch that ended before its first marker kept the first pass from being made: no want was written")
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B2a: without a credential the program does not start, and says why.
|
||||
@@ -112,61 +57,3 @@ func TestNoCredential(t *testing.T) {
|
||||
t.Errorf("run without a credential = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingJS opens every bucket but one, which it refuses with err.
|
||||
type refusingJS struct {
|
||||
jetstream.JetStream
|
||||
missing string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
if name == r.missing {
|
||||
return nil, r.err
|
||||
}
|
||||
return &memKV{values: map[string][]byte{}}, nil
|
||||
}
|
||||
|
||||
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
|
||||
// closed; a put on it is said on wrote.
|
||||
type watchKV struct {
|
||||
*memKV
|
||||
entries []kvEntry
|
||||
closed bool
|
||||
wrote chan string
|
||||
}
|
||||
|
||||
func (w *watchKV) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
ch := make(chan jetstream.KeyValueEntry, len(w.entries)+1)
|
||||
if w.closed {
|
||||
close(ch)
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
for _, e := range w.entries {
|
||||
ch <- e
|
||||
}
|
||||
ch <- nil
|
||||
return keyWatcher{ch}, nil
|
||||
}
|
||||
|
||||
func (w *watchKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if w.wrote != nil {
|
||||
w.wrote <- key
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
type keyWatcher struct{ ch chan jetstream.KeyValueEntry }
|
||||
|
||||
func (k keyWatcher) Updates() <-chan jetstream.KeyValueEntry { return k.ch }
|
||||
func (k keyWatcher) Stop() error { return nil }
|
||||
|
||||
type kvEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e kvEntry) Key() string { return e.key }
|
||||
func (e kvEntry) Value() []byte { return e.value }
|
||||
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
|
||||
@@ -3,8 +3,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -30,29 +28,11 @@ type passer struct {
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
asks *composeAsks
|
||||
// now is the clock; nil is time.Now.
|
||||
now func() time.Time
|
||||
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||
// first pass after a start, which clears what a previous process may have raised (startClears).
|
||||
neither map[string]neitherOne
|
||||
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
|
||||
acts *putActs
|
||||
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
|
||||
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
|
||||
noWant map[string]string
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
if p.now != nil {
|
||||
return p.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
p.compositions(ctx)
|
||||
p.wants(ctx)
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
}
|
||||
@@ -98,8 +78,6 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
wanted := p.w.wantsOf()
|
||||
puts := p.putsByModule()
|
||||
p.staleAsks(ctx, trunk)
|
||||
names := map[string]bool{}
|
||||
for name := range trunk {
|
||||
names[name] = true
|
||||
@@ -107,23 +85,11 @@ func (p *passer) wants(ctx context.Context) {
|
||||
for name := range wanted {
|
||||
names[name] = true
|
||||
}
|
||||
neither := map[string]neitherOne{}
|
||||
// The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay
|
||||
// raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every
|
||||
// assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is
|
||||
// quiet at the router.
|
||||
first := p.neither == nil
|
||||
stale := map[string]neitherOne{}
|
||||
defer func() { p.sayNeither(neither, stale) }()
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name],
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||
GrantedAt: func(fp string) bool {
|
||||
key, err := delivery.ControllerGrantKey(name, fp)
|
||||
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||
},
|
||||
Restored: func(node string) bool {
|
||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||
return err == nil && p.answeredWith(key, assign.OptionRestore)
|
||||
@@ -137,13 +103,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||
hold, _ := delivery.HoldAskKey(name, fp)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
}
|
||||
}
|
||||
if m.Released {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
@@ -153,14 +113,6 @@ func (p *passer) wants(ctx context.Context) {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
}
|
||||
out := assign.Wants(m, nodes, wanted[name], last)
|
||||
gatherNeither(neither, name, out)
|
||||
p.sayNoWant(name, m.Nodes, wanted[name], out)
|
||||
if first {
|
||||
startClears(stale, m, out)
|
||||
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
}
|
||||
if out.Held != nil {
|
||||
key, err := delivery.HeldKey(name, fp)
|
||||
if err == nil {
|
||||
@@ -168,11 +120,8 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
||||
}
|
||||
}
|
||||
if out.Lifted {
|
||||
p.lift(ctx, name, fp)
|
||||
}
|
||||
if out.Held != nil || (m.Held && !out.Lifted) {
|
||||
p.holdAsk(ctx, name, fp, out.Held)
|
||||
if out.Held != nil || m.Held {
|
||||
p.holdAsk(ctx, name, fp)
|
||||
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
||||
}
|
||||
for _, node := range out.Restore {
|
||||
@@ -193,110 +142,6 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// sayNoWant says in the journal why a node of module gets no want after this pass, from the rule the outcome names for it
|
||||
// (issue 1945: a new assignment whose trunk build was not done was held back with nothing said), once per reason, and says
|
||||
// when the node has a want again. A node with a live want, written before and kept or written in this pass, has one.
|
||||
func (p *passer) sayNoWant(module string, nodes []string, before map[string]delivery.Want, out assign.Outcome) {
|
||||
if p.noWant == nil {
|
||||
p.noWant = map[string]string{}
|
||||
}
|
||||
assigned := map[string]bool{}
|
||||
for _, node := range nodes {
|
||||
assigned[node] = true
|
||||
key := node + "." + module
|
||||
w, written := out.Wants[node]
|
||||
if !written {
|
||||
w, written = before[node]
|
||||
}
|
||||
reason := ""
|
||||
if h, held := out.Rules[node]; held && (!written || w.Removed) {
|
||||
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
last, said := p.noWant[key]
|
||||
switch {
|
||||
case reason != "" && reason != last:
|
||||
p.logf("%s on %s gets no want: %s", module, node, reason)
|
||||
p.noWant[key] = reason
|
||||
case reason == "" && said && written && !w.Removed:
|
||||
p.logf("%s on %s has a want", module, node)
|
||||
delete(p.noWant, key)
|
||||
}
|
||||
}
|
||||
for key := range p.noWant {
|
||||
if node, m, _ := strings.Cut(key, "."); m == module && !assigned[node] {
|
||||
delete(p.noWant, key) // no longer assigned there: its removed want says so
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||
type neitherOne struct {
|
||||
node, module string
|
||||
holding assign.Holding
|
||||
}
|
||||
|
||||
// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module.
|
||||
func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) {
|
||||
for node, h := range out.Neither {
|
||||
into[node+"."+module] = neitherOne{node: node, module: module, holding: h}
|
||||
}
|
||||
}
|
||||
|
||||
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||
// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router.
|
||||
func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if _, on := out.Neither[node]; !on {
|
||||
into[node+"."+m.Name] = neitherOne{node: node, module: m.Name}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears,
|
||||
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||
// each is cleared once.
|
||||
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||
if p.neither == nil {
|
||||
p.neither = map[string]neitherOne{}
|
||||
for key, one := range stale {
|
||||
if _, on := now[key]; !on {
|
||||
p.neither[key] = one
|
||||
}
|
||||
}
|
||||
}
|
||||
for key, one := range now {
|
||||
if _, said := p.neither[key]; said {
|
||||
continue
|
||||
}
|
||||
c := assign.NeitherCondition(one.node, one.module, one.holding, true)
|
||||
if p.say != nil {
|
||||
if err := p.say(c); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||
p.neither[key] = one
|
||||
}
|
||||
for key, one := range p.neither {
|
||||
if _, still := now[key]; still {
|
||||
continue
|
||||
}
|
||||
if p.say != nil {
|
||||
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.logf("%s on %s has a want again", one.module, one.node)
|
||||
delete(p.neither, key)
|
||||
}
|
||||
}
|
||||
|
||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
|
||||
func (p *passer) compositions(ctx context.Context) {
|
||||
@@ -365,148 +210,15 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
||||
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
||||
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
||||
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
||||
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
|
||||
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
|
||||
key, err := delivery.HoldAskKey(module, fp)
|
||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||
return
|
||||
}
|
||||
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
|
||||
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
|
||||
now := p.clock()
|
||||
asking := int(now.Unix())
|
||||
h := p.heldHold(module, fp, fresh)
|
||||
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
|
||||
a := assign.HoldAsk(key, module, fp, time.Now())
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
|
||||
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
|
||||
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
|
||||
func (p *passer) lift(ctx context.Context, module, fp string) {
|
||||
held, err := delivery.HeldKey(module, fp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
ask, err := delivery.HoldAskKey(module, fp)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c := assign.LiftedCondition(module, fp)
|
||||
if !p.said[c.Key+".lifted"] {
|
||||
if p.say != nil {
|
||||
if err := p.say(c); err != nil {
|
||||
// The hold stays written, so the next pass lifts it again and says so then.
|
||||
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
p.logf("%s", assign.LiftedWords(module, fp))
|
||||
p.said[c.Key+".lifted"] = true
|
||||
delete(p.said, c.Key+".raised") // a hold written again is said again
|
||||
}
|
||||
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
||||
// The restore asks of that fingerprint, on every node, go with the hold: every node runs it healthy, so no node waits to be
|
||||
// set back from it (issue 1906, note 28).
|
||||
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||
if k, err := delivery.ParseAnyAskKey(key); err == nil && k.Kind == delivery.AskKindRestore && k.Module == module && k.FP == fp {
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// staleAsk says whether the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
|
||||
// behind: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint, or a composition the
|
||||
// controller no longer composes for the node at that fingerprint. A backup ask and an ask of another kind are never stale here.
|
||||
// A module of the trunk whose fingerprint is not read yet, and a composition not read yet, are not stale: nothing known is
|
||||
// nothing taken away.
|
||||
func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
|
||||
k, err := delivery.ParseAnyAskKey(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
switch k.Kind {
|
||||
case delivery.AskKindHold, delivery.AskKindRestore:
|
||||
m, on := trunk[k.Module]
|
||||
return !on || (m.FP2 != "" && m.FP2 != k.FP)
|
||||
case delivery.AskKindRecord:
|
||||
m, on := trunk[k.Module]
|
||||
if !on {
|
||||
return true
|
||||
}
|
||||
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
|
||||
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
|
||||
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
|
||||
// trunk's, its want's, its have's), and only when every one of them is read.
|
||||
for _, fp := range p.fingerprintsOf(k.Node, k.Module, m.FP2) {
|
||||
d, ok := p.w.composed(k.Node, k.Module, fp)
|
||||
if !ok || d.Digest == k.Composed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p.fingerprintsOf(k.Node, k.Module, m.FP2)) > 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fingerprintsOf are the fingerprints a node may still want of module: the trunk's, the one its want names and the one its have
|
||||
// names, each once.
|
||||
func (p *passer) fingerprintsOf(node, module, trunkFP string) []string {
|
||||
var out []string
|
||||
add := func(fp string) {
|
||||
if fp != "" && !slices.Contains(out, fp) {
|
||||
out = append(out, fp)
|
||||
}
|
||||
}
|
||||
add(trunkFP)
|
||||
if key, err := delivery.WantKey(node, module); err == nil {
|
||||
if w, err := delivery.DecodeWant(p.w.value(delivery.AssignerWantsBucket, key)); err == nil {
|
||||
add(w.FP)
|
||||
}
|
||||
}
|
||||
if key, err := delivery.HaveKey(node, module); err == nil {
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil {
|
||||
add(h.FP)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// staleAsks takes away, at every pass, each hold, restore and record ask that staleAsk names (issue 1906, note 28): an ask
|
||||
// outlived its fingerprint, and the teller takes its router ask back from the operator when the key goes. With the trunk not
|
||||
// read at all (no module), nothing is taken away.
|
||||
func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module) {
|
||||
if len(trunk) == 0 {
|
||||
return
|
||||
}
|
||||
for key := range p.w.read(delivery.AssignerAsksBucket) {
|
||||
if p.staleAsk(key, trunk) {
|
||||
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
|
||||
p.remove(ctx, delivery.AssignerAsksBucket, key)
|
||||
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
|
||||
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
|
||||
// raised again if the trunk returns to the fingerprint.
|
||||
if k, err := delivery.ParseAnyAskKey(key); err == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore) {
|
||||
p.clearHeld(k.Module, k.FP)
|
||||
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
||||
// pass.
|
||||
func (p *passer) remove(ctx context.Context, bucket, key string) {
|
||||
if !p.w.has(bucket, key) {
|
||||
return
|
||||
}
|
||||
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
|
||||
p.logf("%s %s could not be taken away: %v", bucket, key, err)
|
||||
return
|
||||
}
|
||||
p.w.put(bucket, key, nil)
|
||||
}
|
||||
|
||||
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
||||
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
||||
if fresh != nil {
|
||||
@@ -533,8 +245,6 @@ func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *de
|
||||
return
|
||||
}
|
||||
p.said[c.Key+".raised"] = true
|
||||
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
|
||||
delete(p.said, c.Key+".cleared") // and released again is cleared again
|
||||
}
|
||||
|
||||
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
||||
@@ -548,21 +258,11 @@ func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
|
||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||
}
|
||||
|
||||
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
|
||||
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
|
||||
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
|
||||
// the same fingerprint, does not carry over to a later one (issue 1906).
|
||||
// answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the
|
||||
// operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19).
|
||||
func (p *passer) answeredWith(key, option string) bool {
|
||||
a, ok := p.w.answer(key)
|
||||
if !ok || a.Option != option {
|
||||
return false
|
||||
}
|
||||
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
id, _, _ := strings.Cut(a.Warrant, "@")
|
||||
return id == ask.Ask.ID
|
||||
return ok && a.Option == option
|
||||
}
|
||||
|
||||
// clearHeld clears, once, the condition of a hold the operator released.
|
||||
|
||||
@@ -1,685 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// memKV is the part of a bucket a pass writes: puts and deletes.
|
||||
type memKV struct {
|
||||
jetstream.KeyValue
|
||||
values map[string][]byte
|
||||
}
|
||||
|
||||
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.values[key] = value
|
||||
return uint64(len(m.values)), nil
|
||||
}
|
||||
|
||||
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||
delete(m.values, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
|
||||
func newPasser(kvs buckets, at time.Time) *passer {
|
||||
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||
now: func() time.Time { return at }}
|
||||
}
|
||||
|
||||
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
|
||||
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
|
||||
// an id it keeps).
|
||||
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
held := &delivery.Held{Node: "ace", Words: "health failed"}
|
||||
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
|
||||
first.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first.lift(context.Background(), "web", fpTest)
|
||||
if _, there := asksKV.values[key]; there {
|
||||
t.Fatal("the lift left the hold ask")
|
||||
}
|
||||
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
|
||||
again.holdAsk(context.Background(), "web", fpTest, held)
|
||||
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a2.Ask.ID == a1.Ask.ID {
|
||||
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
||||
// taken away, is not taken for the hold that stands now.
|
||||
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
put := func(asking int) string {
|
||||
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
return a.Ask.ID
|
||||
}
|
||||
first := put(100)
|
||||
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
|
||||
p.w.put(delivery.AnswersBucket, key, v)
|
||||
if !p.answeredWith(key, "release") {
|
||||
t.Fatal("the answer to the ask that stands is not taken")
|
||||
}
|
||||
if p.answeredWith(key, "keep") {
|
||||
t.Error("an answer was taken for an option not chosen")
|
||||
}
|
||||
put(200) // the same fingerprint held again, after a lift
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("the answer to an earlier hold carried over to a later one")
|
||||
}
|
||||
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
|
||||
if p.answeredWith(key, "release") {
|
||||
t.Error("an answer to an ask taken away was taken")
|
||||
}
|
||||
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||
}
|
||||
|
||||
// memEntry is a value a memKV holds, as a bucket's Get answers it.
|
||||
type memEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
value []byte
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (e memEntry) Value() []byte { return e.value }
|
||||
func (e memEntry) Revision() uint64 { return e.revision }
|
||||
func (m *memKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
v, ok := m.values[key]
|
||||
if !ok {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return memEntry{value: v, revision: uint64(len(v))}, nil
|
||||
}
|
||||
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if uint64(len(m.values[key])) != revision {
|
||||
return 0, errors.New("wrong last sequence")
|
||||
}
|
||||
m.values[key] = value
|
||||
return uint64(len(value)), nil
|
||||
}
|
||||
|
||||
// memStore is the runtime's states for the put's tools, over the same memKVs a pass writes.
|
||||
type memStore map[string]*memKV
|
||||
|
||||
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := s[state].values[key]
|
||||
return v, uint64(len(v)), ok, nil
|
||||
}
|
||||
func (s memStore) put(state, key string, value []byte) error {
|
||||
s[state].values[key] = value
|
||||
return nil
|
||||
}
|
||||
func (s memStore) keys(state string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range s[state].values {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fakeBus is the assigner's process account in a test: every call recorded, each answered at once.
|
||||
type fakeBus struct {
|
||||
calls chan string
|
||||
events []string
|
||||
}
|
||||
|
||||
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||
b.calls <- fmt.Sprintf("%s %v", key, body)
|
||||
return json.RawMessage(`{"ok":true,"output":""}`), nil
|
||||
}
|
||||
func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
return b.Call(node+"/"+key, body)
|
||||
}
|
||||
func (b *fakeBus) Emit(eventType string, body any) error {
|
||||
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant
|
||||
// the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have
|
||||
// the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release
|
||||
// gives the want back to the rules. Each step is said as the event put.
|
||||
func putOfTheBus(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
|
||||
puts, asksKV, wantsKV := mem(), mem(), mem()
|
||||
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||
p := newPasser(kvs, now)
|
||||
bus := &fakeBus{calls: make(chan string, 8)}
|
||||
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
|
||||
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
|
||||
p.acts.snapshot = func(context.Context, string, string) (string, error) {
|
||||
return "novox's restore point of nats (streams) taken 22:01", nil
|
||||
}
|
||||
// The world: nats assigned to novox, built at fp, the bus by its manifest, novox running it from before the switch.
|
||||
world := func(bucket, key string, v any) {
|
||||
var data []byte
|
||||
switch x := v.(type) {
|
||||
case []byte:
|
||||
data = x
|
||||
default:
|
||||
data, _ = json.Marshal(x)
|
||||
}
|
||||
p.w.put(bucket, key, data)
|
||||
}
|
||||
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||
world(delivery.ModulesBucket, "nats", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||
bkey, _ := delivery.BuilderBuildKey("nats", fp)
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
|
||||
world(delivery.BuilderBuildsBucket, bkey, built)
|
||||
hkey, _ := delivery.HaveKey("novox", "nats")
|
||||
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
|
||||
builds := mem()
|
||||
builds.values[bkey] = built
|
||||
modules := mem()
|
||||
modules.values["nats"] = mod
|
||||
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
|
||||
s.now = func() time.Time { return now }
|
||||
was := answerWithin
|
||||
answerWithin = 0
|
||||
defer func() { answerWithin = was }()
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
t.Error("a prefix of a fingerprint the builder never built was taken")
|
||||
}
|
||||
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
|
||||
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
|
||||
}
|
||||
if got, err := s.version("nats", "newest"); err != nil || got != fp {
|
||||
t.Errorf("newest reads %q, %v", got, err)
|
||||
}
|
||||
ans, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "the bus's user list"}, "shanks.node-tools")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
|
||||
t.Fatalf("put answered %+v", a)
|
||||
}
|
||||
pkey, _ := delivery.PutKey("novox", "nats")
|
||||
step := func() delivery.Put {
|
||||
t.Helper()
|
||||
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
|
||||
p.pass(ctx)
|
||||
got, err := delivery.DecodePut(puts.values[pkey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
|
||||
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
select {
|
||||
case call := <-bus.calls:
|
||||
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||
default:
|
||||
}
|
||||
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||
world(delivery.ControllerGrantsBucket, gkey, g)
|
||||
got := step()
|
||||
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||
if got.Step != delivery.PutPerson || got.Ask != akey {
|
||||
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil || a.Ask.Options[0].Level != asks.Approve {
|
||||
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
|
||||
}
|
||||
if _, there := wantsKV.values["want.novox.nats"]; there {
|
||||
t.Fatal("the bus's want was written before the operator's warrant")
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
got = step()
|
||||
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
|
||||
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||
}
|
||||
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
|
||||
t.Errorf("the warrant was recorded as %q", call)
|
||||
}
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
|
||||
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
world(delivery.HaveBucket, hkey, running)
|
||||
if got := step(); got.Step != delivery.PutHolding {
|
||||
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := step(); got.Step != delivery.PutReleased {
|
||||
t.Fatalf("released, the put stands at %s", got.Step)
|
||||
}
|
||||
listed, _ := s.doPuts(map[string]any{"module": "nats"})
|
||||
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
|
||||
t.Errorf("puts lists %+v", l)
|
||||
}
|
||||
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
|
||||
t.Errorf("the steps said as events: %v", bus.events)
|
||||
}
|
||||
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
|
||||
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
|
||||
world(delivery.HaveBucket, hkey, before)
|
||||
s.now = func() time.Time { return now.Add(time.Minute) }
|
||||
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := step()
|
||||
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
|
||||
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
|
||||
}
|
||||
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
|
||||
world(delivery.AnswersBucket, akey, yes2)
|
||||
if got := step(); got.Step != delivery.PutSnapshot {
|
||||
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
|
||||
}
|
||||
<-woke
|
||||
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
|
||||
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
|
||||
wantsKV.values["want.novox.nats"] = oldWant
|
||||
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
|
||||
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.pass(ctx)
|
||||
if string(wantsKV.values["want.novox.nats"]) != string(oldWant) {
|
||||
t.Errorf("a want was written for a put released meanwhile: %s", wantsKV.values["want.novox.nats"])
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
|
||||
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||
// the journal, so it is loud with no operator channel.
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
failing := true
|
||||
p.say = func(c asks.Condition) error {
|
||||
if failing {
|
||||
failing = false
|
||||
return errors.New("the channel is away")
|
||||
}
|
||||
said = append(said, c)
|
||||
return nil
|
||||
}
|
||||
var logged []string
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||
p.sayNeither(one, nil) // the say fails
|
||||
p.sayNeither(one, nil)
|
||||
p.sayNeither(one, nil)
|
||||
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||
}
|
||||
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
p.sayNeither(map[string]neitherOne{}, nil)
|
||||
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
got := map[string]neitherOne{}
|
||||
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||
gatherNeither(got, "rofi", assign.Outcome{})
|
||||
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||
t.Errorf("gathered = %+v", got)
|
||||
}
|
||||
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
|
||||
// when it has a want again: one line per change, not per pass.
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
|
||||
var logged []string
|
||||
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
|
||||
wanted := assign.Outcome{Wants: map[string]delivery.Want{"shanks": {FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
|
||||
for i, c := range []struct {
|
||||
before map[string]delivery.Want
|
||||
out assign.Outcome
|
||||
line string // the line said in this pass, "" for none
|
||||
}{
|
||||
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
|
||||
{nil, notDone, ""},
|
||||
{map[string]delivery.Want{"shanks": {Removed: true}}, notDone, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
if got := strings.Join(logged, "\n"); got != c.line {
|
||||
t.Errorf("pass %d said %q, want %q", i, got, c.line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
p.w.put(delivery.HaveBucket, "g14.web", have)
|
||||
key, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
p.w.put(delivery.AssignerWantsBucket, key, want)
|
||||
p.wants(context.Background())
|
||||
cleared := map[string]bool{}
|
||||
for _, c := range said {
|
||||
if c.State != asks.StateCleared {
|
||||
t.Errorf("the first pass raised %+v", c)
|
||||
}
|
||||
cleared[c.Key] = true
|
||||
}
|
||||
neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key
|
||||
held := assign.HeldCondition("web", fpTest, "", "", false, false).Key
|
||||
if !cleared[neither] || !cleared[held] {
|
||||
t.Errorf("cleared = %v, want %s and %s", cleared, neither, held)
|
||||
}
|
||||
n := len(said)
|
||||
p.wants(context.Background())
|
||||
if len(said) != n {
|
||||
t.Errorf("a second pass said again: %+v", said[n:])
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||
// raised (the first pass after a start), each once.
|
||||
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||
setup := func() (*passer, *[]asks.Condition) {
|
||||
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
said := &[]asks.Condition{}
|
||||
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
|
||||
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
|
||||
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
|
||||
p.w.put(delivery.AssignerWantsBucket, hkey, held)
|
||||
return p, said
|
||||
}
|
||||
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
|
||||
neithers := func(said []asks.Condition) (raised, cleared int) {
|
||||
for _, c := range said {
|
||||
if c.Key != key {
|
||||
continue
|
||||
}
|
||||
if c.State == asks.StateRaised {
|
||||
raised++
|
||||
} else {
|
||||
cleared++
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
// a standing neither this process raised
|
||||
p, said := setup()
|
||||
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: delivery.RuleNotHeld}}}
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||
t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||
}
|
||||
// one a previous process raised: the first pass after a start clears it
|
||||
p, said = setup()
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906 note 28: an ask outlives its fingerprint. Each pass takes away the hold, restore and record asks of a module whose
|
||||
// fingerprint (or composition) is not the trunk's now, or whose module left the trunk; a trunk not read takes nothing away; a
|
||||
// backup ask is left; and a lift takes the restore asks of its fingerprint on every node.
|
||||
func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
|
||||
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("a", 64), delivery.SourcePrefix2+strings.Repeat("c", 64)
|
||||
digOld, digNew := "sha256:"+strings.Repeat("d", 64), "sha256:"+strings.Repeat("e", 64)
|
||||
trunkOf := func(fp string) map[string]delivery.Module {
|
||||
return map[string]delivery.Module{"web": {FP2: fp, Manifest: "sha256:" + strings.Repeat("1", 64), Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}}
|
||||
}
|
||||
type ask struct{ name, key string }
|
||||
holdOld, _ := delivery.HoldAskKey("web", fpOld)
|
||||
holdNew, _ := delivery.HoldAskKey("web", fpNew)
|
||||
holdGone, _ := delivery.HoldAskKey("gone", fpOld)
|
||||
restOld, _ := delivery.RestoreAskKey("novox", "web", fpOld)
|
||||
restNew, _ := delivery.RestoreAskKey("novox", "web", fpNew)
|
||||
recOld, _ := delivery.RecordAskKey("novox", "web", digOld)
|
||||
recNew, _ := delivery.RecordAskKey("novox", "web", digNew)
|
||||
backOld, _ := delivery.BackupAskKey("novox", "web", fpOld)
|
||||
all := []string{holdOld, holdNew, holdGone, restOld, restNew, recOld, recNew, backOld}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
trunk map[string]delivery.Module
|
||||
gone []string // the keys taken away
|
||||
noDigit bool // the controller's composition is not read
|
||||
heldAt string // the node's want is at this fingerprint, whose composition is digOld
|
||||
}{
|
||||
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, ""},
|
||||
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, ""},
|
||||
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, ""},
|
||||
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, ""},
|
||||
{"a node held back at the old fingerprint is still asked about its composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, false, fpOld},
|
||||
{"the trunk is not read at all", nil, nil, false, ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
var said []asks.Condition
|
||||
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||
for _, key := range all {
|
||||
k, _ := delivery.ParseAnyAskKey(key)
|
||||
var a delivery.AssignerAsk
|
||||
switch k.Kind {
|
||||
case delivery.AskKindHold:
|
||||
a = assign.HoldAsk(key, k.Module, k.FP, "novox", "w", 1, time.Now())
|
||||
case delivery.AskKindRestore:
|
||||
a = assign.RestoreAsk(key, k.Node, k.Module, k.FP, "w", time.Now())
|
||||
case delivery.AskKindRecord:
|
||||
a = assign.RecordAsk(key, k.Node, k.Module, fpNew, assign.RecordNeed{Node: k.Node, Digest: k.Composed}, time.Now())
|
||||
default:
|
||||
a = assign.BackupAsk(key, k.Node, k.Module, k.FP, digOld, "w", 2, time.Now())
|
||||
}
|
||||
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksKV.values[key] = v
|
||||
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||
}
|
||||
if !c.noDigit {
|
||||
ckey, _ := delivery.ComposedKey("novox", "web", fpNew)
|
||||
v, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
p.w.put(delivery.ControllerComposedBucket, ckey, v)
|
||||
okey, _ := delivery.ComposedKey("novox", "web", fpOld)
|
||||
old, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digNew, Judgement: delivery.JudgedUnchanged})
|
||||
if c.heldAt != "" {
|
||||
old, _ = delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digOld, Judgement: delivery.JudgedUnchanged})
|
||||
}
|
||||
p.w.put(delivery.ControllerComposedBucket, okey, old)
|
||||
}
|
||||
if c.heldAt != "" {
|
||||
wkey, _ := delivery.WantKey("novox", "web")
|
||||
w, _ := delivery.EncodeWant(delivery.Want{FP: c.heldAt, Composed: digOld})
|
||||
p.w.put(delivery.AssignerWantsBucket, wkey, w)
|
||||
}
|
||||
heldOld, _ := delivery.HeldKey("web", fpOld)
|
||||
wantsKV.values[heldOld] = []byte("x")
|
||||
p.w.put(delivery.AssignerWantsBucket, heldOld, []byte("x"))
|
||||
firstAsk, _ := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
p.staleAsks(context.Background(), c.trunk)
|
||||
// the held record and condition of a fingerprint whose hold ask went are taken away with it
|
||||
heldOld, _ = delivery.HeldKey("web", fpOld)
|
||||
_, heldThere := wantsKV.values[heldOld]
|
||||
cleared := slices.ContainsFunc(said, func(c asks.Condition) bool {
|
||||
return c.Key == assign.HeldCondition("web", fpOld, "", "", false, false).Key && c.State == asks.StateCleared
|
||||
})
|
||||
if gone := slices.Contains(c.gone, holdOld); !heldThere || cleared != gone {
|
||||
t.Errorf("hold ask of the old fingerprint gone = %v: its held record stays = %v (must), its condition cleared = %v", gone, heldThere, cleared)
|
||||
}
|
||||
for _, key := range all {
|
||||
_, there := asksKV.values[key]
|
||||
want := !slices.Contains(c.gone, key)
|
||||
if there != want {
|
||||
t.Errorf("%s: left in the bucket = %v, want %v", key, there, want)
|
||||
}
|
||||
}
|
||||
if slices.Contains(c.gone, holdOld) {
|
||||
// the trunk returns to the fingerprint (a revert): the held record still holds it, and it is asked and said again
|
||||
p.now = func() time.Time { return time.Now().Add(time.Hour) }
|
||||
p.holdAsk(context.Background(), "web", fpOld, nil)
|
||||
p.heldCondition(context.Background(), "web", fpOld, nil, false)
|
||||
again, err := delivery.DecodeAssignerAsk(holdOld, asksKV.values[holdOld])
|
||||
raised := slices.ContainsFunc(said, func(c asks.Condition) bool { return c.State == asks.StateRaised })
|
||||
if err != nil || again.Ask.ID == firstAsk.Ask.ID || !raised {
|
||||
t.Errorf("the trunk returned to the held fingerprint: ask %q (first %q, %v), condition raised again = %v", again.Ask.ID, firstAsk.Ask.ID, err, raised)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
keep, _ := delivery.RestoreAskKey("ace", "web", fpNew)
|
||||
keepOther, _ := delivery.RestoreAskKey("ace", "other", fpOld)
|
||||
for _, key := range []string{restOld, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)), keep, keepOther} {
|
||||
asksKV.values[key] = []byte("x")
|
||||
p.w.put(delivery.AssignerAsksBucket, key, []byte("x"))
|
||||
}
|
||||
p.lift(context.Background(), "web", fpOld)
|
||||
for key, gone := range map[string]bool{restOld: true, mustKey(delivery.RestoreAskKey("ace", "web", fpOld)): true, keep: false, keepOther: false} {
|
||||
if _, there := asksKV.values[key]; there == gone {
|
||||
t.Errorf("%s: left in the bucket = %v after the lift", key, there)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func mustKey(key string, err error) string {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// 1906 note 28, review: a node held back at an earlier fingerprint is asked about the revised composition of that fingerprint
|
||||
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
|
||||
// yes the want names the revised digest.
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 back at fpOld
|
||||
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||
put(delivery.PresenceBucket, "g14", on)
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
put(delivery.HaveBucket, "g14.web", have)
|
||||
wkey, _ := delivery.WantKey("g14", "web")
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
|
||||
wantsKV.values[wkey] = want
|
||||
put(delivery.AssignerWantsBucket, wkey, want)
|
||||
ckey, _ := delivery.ComposedKey("g14", "web", fpOld)
|
||||
cd, cerr := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord, PolicyFrom: "default", Against: was, Words: "recreates"})
|
||||
if cerr != nil {
|
||||
t.Fatal(cerr)
|
||||
}
|
||||
put(delivery.ControllerComposedBucket, ckey, cd)
|
||||
tkey, _ := delivery.ComposedKey("g14", "web", fpTest) // the trunk's own composition, another digest than the revised one
|
||||
tcd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("f", 64), Judgement: delivery.JudgedUnchanged})
|
||||
put(delivery.ControllerComposedBucket, tkey, tcd)
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", revised)
|
||||
for i := 0; i < 2; i++ {
|
||||
p.wants(context.Background())
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Fatalf("pass %d: the record ask of the held-back node is not there", i+1)
|
||||
}
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T01:00:00Z", Words: "the operator"})
|
||||
put(delivery.AnswersBucket, akey, yes)
|
||||
p.wants(context.Background())
|
||||
got, err := delivery.DecodeWant(wantsKV.values[wkey])
|
||||
if err != nil || got.FP != fpOld || got.Composed != revised {
|
||||
t.Errorf("after the yes the want is %+v (%v), want it at %s naming %s", got, err, fpOld, revised)
|
||||
}
|
||||
if _, there := asksKV.values[akey]; !there {
|
||||
t.Error("the answered record ask was taken away")
|
||||
}
|
||||
}
|
||||
@@ -1,449 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The puts (novox/hq ADR 0546): the tool `assigner_put` writes the key `<node>.<module>` of the assigner's puts, and each pass
|
||||
// takes every put that has not ended one step further (assign.StepPut), doing the acts the step asks for: the put's ask, the
|
||||
// bus's snapshot, the warranted record. Each act's outcome is heard back as a change of the world (the composition, the grant,
|
||||
// the answer) or, for the calls, as a wake once they answer; nothing polls but the snapshot's read-back of the backup holder,
|
||||
// as the controller's bus step reads it (ADR 0235), bounded. Every step is said as the event mesh-delivery-assigner.put, so a
|
||||
// caller waits on it.
|
||||
|
||||
// PutEvent is the event a put's every step is said as: mesh.mod.mesh-delivery-assigner.event.put.
|
||||
const PutEvent = "put"
|
||||
|
||||
// caller is what the puts need of the bus as the assigner's own process account: a call of a seat's verb, on one node or on
|
||||
// the seat held once, and an event said. process.Conn, or a test's.
|
||||
type caller interface {
|
||||
Call(key string, body any) (json.RawMessage, error)
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
}
|
||||
|
||||
// putActs keeps, by put (its key and when it was asked), what the acts of a put came to while this process runs: a restart
|
||||
// takes the snapshot again, without harm.
|
||||
type putActs struct {
|
||||
mu sync.Mutex
|
||||
bus caller
|
||||
wake func()
|
||||
logf func(string, ...any)
|
||||
snapping map[string]bool
|
||||
snapped map[string]string
|
||||
snapNo map[string]string
|
||||
warranted map[string]bool
|
||||
warrantNo map[string]string
|
||||
said map[string]string
|
||||
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||
}
|
||||
|
||||
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||
said: map[string]string{}}
|
||||
a.snapshot = a.snapshotTheBus
|
||||
return a
|
||||
}
|
||||
|
||||
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
|
||||
func identity(key string, p delivery.Put) string {
|
||||
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// putsByModule is every put the assigner holds, by module then node.
|
||||
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
|
||||
out := map[string]map[string]delivery.Put{}
|
||||
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
put, err := delivery.DecodePut(value)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read, and is left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]delivery.Put{}
|
||||
}
|
||||
out[module][node] = put
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// puts takes every put that has not ended one step further.
|
||||
func (p *passer) puts(ctx context.Context) {
|
||||
if p.acts == nil {
|
||||
return
|
||||
}
|
||||
all := p.putsByModule()
|
||||
if len(all) == 0 {
|
||||
return
|
||||
}
|
||||
assigned, _ := p.w.assigned()
|
||||
trunk := p.w.trunk()
|
||||
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||
wanted := p.w.wantsOf()
|
||||
modules := make([]string, 0, len(all))
|
||||
for m := range all {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
var manifest catalogue.Manifest
|
||||
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
|
||||
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
|
||||
manifest = got.manifest
|
||||
}
|
||||
}
|
||||
for node, put := range all[module] {
|
||||
if delivery.PutEnded(put.Step) {
|
||||
continue
|
||||
}
|
||||
key, _ := delivery.PutKey(node, module)
|
||||
id := identity(key, put)
|
||||
world := p.putWorld(node, module, put, manifest, assigned[node])
|
||||
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
|
||||
next := assign.StepPut(world)
|
||||
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
|
||||
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
|
||||
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
|
||||
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
|
||||
next.Put.Composed = w.Composed
|
||||
}
|
||||
}
|
||||
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
|
||||
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
|
||||
}
|
||||
}
|
||||
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
|
||||
!strings.Contains(next.Put.Words, failed) {
|
||||
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
|
||||
}
|
||||
p.actOn(ctx, node, module, id, world, next)
|
||||
written := samePut(put, next.Put) || p.writePut(ctx, key, put, next.Put)
|
||||
if !written {
|
||||
continue // released or put again meanwhile: nothing is done for the put that no longer stands
|
||||
}
|
||||
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
|
||||
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
|
||||
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||
want := assign.PutWant(next.Put)
|
||||
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
|
||||
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||
}
|
||||
}
|
||||
if next.Warranted {
|
||||
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func normalWant(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
return w
|
||||
}
|
||||
|
||||
// putWorld is what one step of a put reads from the world.
|
||||
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
|
||||
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
|
||||
for _, m := range assignedHere {
|
||||
if m == module {
|
||||
w.Assigned = true
|
||||
}
|
||||
}
|
||||
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
|
||||
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
|
||||
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
|
||||
w.Build, w.BuildWords = b.State, b.Error
|
||||
} else {
|
||||
w.Build = "unreadable: " + err.Error()
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||
if d := w.Composed; d != nil {
|
||||
policy := d.Policy
|
||||
if policy == "" {
|
||||
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||
}
|
||||
w.Recorded = policy == catalogue.PolicyRecord
|
||||
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
|
||||
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||
switch {
|
||||
case p.answeredWith(akey, assign.OptionYes):
|
||||
w.Answer = assign.OptionYes
|
||||
case p.answeredWith(akey, assign.OptionNo):
|
||||
w.Answer = assign.OptionNo
|
||||
default:
|
||||
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if hkey, err := delivery.HaveKey(node, module); err == nil {
|
||||
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
|
||||
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||
w.Have = &h
|
||||
}
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
|
||||
// from their new owner and nothing else changes.
|
||||
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||
var out *delivery.ComposedDigest
|
||||
if d, ok := p.w.composed(node, module, fp); ok {
|
||||
out = &d
|
||||
}
|
||||
granted := false
|
||||
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
|
||||
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||
granted = g.NamesNode(node, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, granted
|
||||
}
|
||||
|
||||
// actOn does the acts a step asks for.
|
||||
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
|
||||
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
|
||||
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
|
||||
// raises it anew under its own id.
|
||||
akey := next.Put.Ask
|
||||
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||
if err != nil || current.Asking != assign.PutAsking(next.Put) {
|
||||
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
|
||||
if err != nil {
|
||||
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
|
||||
} else {
|
||||
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||
}
|
||||
}
|
||||
}
|
||||
if next.Snapshot {
|
||||
p.acts.snap(ctx, id, node, module)
|
||||
}
|
||||
}
|
||||
|
||||
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
|
||||
func (p *passer) performs(key string, act asks.Act) error {
|
||||
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
|
||||
}
|
||||
for _, o := range a.Ask.Options {
|
||||
if o.ID == assign.OptionYes {
|
||||
return o.Performs(act)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the ask %s offers no yes", key)
|
||||
}
|
||||
|
||||
// samePut says two puts say the same: their step, words and what they found.
|
||||
func samePut(a, b delivery.Put) bool {
|
||||
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
|
||||
}
|
||||
|
||||
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
|
||||
// released it since; and says it as the event put. It says whether it wrote.
|
||||
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
|
||||
kv := p.kvs[delivery.AssignerPutsBucket]
|
||||
if kv == nil {
|
||||
return false
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
|
||||
return false
|
||||
}
|
||||
now, err := delivery.DecodePut(e.Value())
|
||||
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
|
||||
return false // asked again or released meanwhile: the next pass reads that
|
||||
}
|
||||
data, err := delivery.EncodePut(next)
|
||||
if err != nil {
|
||||
p.logf("the put %s is not written: %v", key, err)
|
||||
return false
|
||||
}
|
||||
if _, err := kv.Update(ctx, key, data, e.Revision()); err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
p.logf("the put %s could not be written, and is taken further at the next pass: %v", key, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
p.w.put(delivery.AssignerPutsBucket, key, data)
|
||||
p.logf("put %s: %s: %s", key, next.Step, next.Words)
|
||||
p.acts.say(key, next)
|
||||
return true
|
||||
}
|
||||
|
||||
// state is what this process knows of a put's acts.
|
||||
func (a *putActs) state(id string) (snapshot string, snapping bool, snapNo string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.snapped[id], a.snapping[id], a.snapNo[id]
|
||||
}
|
||||
|
||||
// snap has the bus's snapshot taken once for a put, in the background.
|
||||
func (a *putActs) snap(ctx context.Context, id, node, module string) {
|
||||
a.mu.Lock()
|
||||
if a.snapping[id] || a.snapped[id] != "" || a.snapNo[id] != "" {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.snapping[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
defer a.wake()
|
||||
where, err := a.snapshot(ctx, node, module)
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.snapping[id] = false
|
||||
if err != nil {
|
||||
a.snapNo[id] = err.Error()
|
||||
return
|
||||
}
|
||||
a.snapped[id] = where
|
||||
}()
|
||||
}
|
||||
|
||||
// SnapshotWithin is how long the bus node's backup holder is given to take the bus's snapshot (as the controller's bus step).
|
||||
var SnapshotWithin = 15 * time.Minute
|
||||
|
||||
// snapshotLook is how often the backup holder's record is read back while the snapshot is taken.
|
||||
var snapshotLook = 10 * time.Second
|
||||
|
||||
// backedUp is one module as the backup holder's `backed-up` answers it.
|
||||
type backedUp struct {
|
||||
Module string `json:"module"`
|
||||
Data []struct {
|
||||
Item string `json:"item"`
|
||||
LastBackup *time.Time `json:"last_backup"`
|
||||
Error string `json:"error"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// snapshotTheBus asks the bus node's backup holder to back the bus module up now, whose dump is the streams' snapshot
|
||||
// (ADR 0235), and reads its record back until it names a backup newer than the ask: where the snapshot is.
|
||||
func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (string, error) {
|
||||
asked := time.Now()
|
||||
if _, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".now", map[string]any{"module": module}); err != nil {
|
||||
return "", fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||
}
|
||||
deadline := asked.Add(SnapshotWithin)
|
||||
for {
|
||||
if raw, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".backed-up", map[string]any{}); err == nil {
|
||||
var modules []backedUp
|
||||
if json.Unmarshal(raw, &modules) == nil {
|
||||
for _, m := range modules {
|
||||
for _, it := range m.Data {
|
||||
if m.Module == module && it.LastBackup != nil && it.LastBackup.After(asked) && it.Error == "" {
|
||||
return fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, it.Item, it.LastBackup.UTC().Format(time.RFC3339)), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return "", fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s", node, SnapshotWithin)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(snapshotLook):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
|
||||
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||
func (a *putActs) warrant(id, ask string) {
|
||||
a.mu.Lock()
|
||||
if a.warranted[id] {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.warranted[id] = true
|
||||
a.mu.Unlock()
|
||||
go func() {
|
||||
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||
a.mu.Lock()
|
||||
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
|
||||
a.mu.Unlock()
|
||||
a.wake()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// warrantFailed says why the put's warrant was not recorded, "" when it was or is not yet known.
|
||||
func (a *putActs) warrantFailed(id string) string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
return a.warrantNo[id]
|
||||
}
|
||||
|
||||
// putSaid is the body of the event put.
|
||||
type putSaid struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
Hold bool `json:"hold"`
|
||||
Step string `json:"step"`
|
||||
Words string `json:"words"`
|
||||
Composed string `json:"composed,omitempty"`
|
||||
}
|
||||
|
||||
// say says a put's step as the event put, once per step and words.
|
||||
func (a *putActs) say(key string, p delivery.Put) {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || a.bus == nil {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if a.said[key] == p.Step+p.Words {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
a.said[key] = p.Step + p.Words
|
||||
a.mu.Unlock()
|
||||
if err := a.bus.Emit(PutEvent, putSaid{Node: node, Module: module, FP: p.FP, Hold: p.Hold, Step: p.Step, Words: p.Words, Composed: p.Composed}); err != nil {
|
||||
a.logf("the put %s's step %s is not said as an event: %v", key, p.Step, err)
|
||||
}
|
||||
}
|
||||
|
||||
// shortHex is a digest's first twelve hex digits.
|
||||
func shortHex(digest string) string {
|
||||
h := strings.TrimPrefix(digest, "sha256:")
|
||||
if len(h) > 12 {
|
||||
return h[:12]
|
||||
}
|
||||
return h
|
||||
}
|
||||
@@ -1,364 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
|
||||
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
|
||||
// answerWithin is how long assigner_put waits for the assigner's process to take the put up before it answers what stands.
|
||||
var answerWithin = 20 * time.Second
|
||||
|
||||
// store is what the tools need of the states: the runtime's (stdio.State), or a test's.
|
||||
type store interface {
|
||||
get(state, key string) ([]byte, uint64, bool, error)
|
||||
put(state, key string, value []byte) error
|
||||
keys(state string) ([]string, error)
|
||||
}
|
||||
|
||||
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
|
||||
type putTools struct {
|
||||
st store
|
||||
now func() time.Time
|
||||
mu sync.Mutex
|
||||
// heard is every put the watch handed, by key; changed is closed and replaced at each.
|
||||
heard map[string][]byte
|
||||
changed chan struct{}
|
||||
}
|
||||
|
||||
func newPutTools(st store) *putTools {
|
||||
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
// hear takes a put's change in from the watch.
|
||||
func (s *putTools) hear(key string, value []byte) {
|
||||
s.mu.Lock()
|
||||
s.heard[key] = value
|
||||
close(s.changed)
|
||||
s.changed = make(chan struct{})
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
|
||||
var (
|
||||
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
|
||||
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
|
||||
)
|
||||
|
||||
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
|
||||
// of one the builder built, which must name one build alone.
|
||||
func (s *putTools) version(module, version string) (string, error) {
|
||||
v := strings.ToLower(strings.TrimSpace(version))
|
||||
switch {
|
||||
case v == "newest":
|
||||
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
|
||||
}
|
||||
m, err := delivery.DecodeModule(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
|
||||
}
|
||||
if m.FP2 == "" {
|
||||
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
|
||||
}
|
||||
return m.FP2, nil
|
||||
case fullFP.MatchString(v):
|
||||
return v, nil
|
||||
case prefix.MatchString(v):
|
||||
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
|
||||
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
|
||||
}
|
||||
var found []string
|
||||
for _, k := range keys {
|
||||
m, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
|
||||
found = append(found, fp)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
sort.Strings(found)
|
||||
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
|
||||
}
|
||||
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
|
||||
}
|
||||
|
||||
// putAnswer is what assigner_put and assigner_puts answer of one put.
|
||||
type putAnswer struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
delivery.Put
|
||||
Follow string `json:"follow,omitempty"`
|
||||
}
|
||||
|
||||
// doPut is the tool assigner_put.
|
||||
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("put says why: it is kept with the put and said in any ask")
|
||||
}
|
||||
hold := false
|
||||
switch arg(args, "hold") {
|
||||
case "true":
|
||||
hold = true
|
||||
case "", "false":
|
||||
default:
|
||||
return nil, fmt.Errorf("hold is true or false, not %q", arg(args, "hold"))
|
||||
}
|
||||
fp, err := s.version(module, arg(args, "version"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now().UTC()
|
||||
words := fmt.Sprintf("asked by %s: %s", by, why)
|
||||
if raw, _, found, err := s.st.get(putsState, key); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
} else if found {
|
||||
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
|
||||
if before.FP == fp && before.Hold == hold {
|
||||
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
|
||||
}
|
||||
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
|
||||
}
|
||||
}
|
||||
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
|
||||
}
|
||||
|
||||
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
|
||||
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
|
||||
deadline := time.After(answerWithin)
|
||||
for {
|
||||
s.mu.Lock()
|
||||
raw, changed := s.heard[key], s.changed
|
||||
s.mu.Unlock()
|
||||
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
|
||||
p = now
|
||||
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
|
||||
return p // it waits on a person or the node, or ended
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-changed:
|
||||
case <-deadline:
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// followPut says how a caller follows a put after the answer.
|
||||
func followPut(node, module string) string {
|
||||
return fmt.Sprintf("the put goes on: mesh_wait on mesh-delivery-assigner.put with filter {\"node\": %q, \"module\": %q}, then "+
|
||||
"mesh-delivery-assigner.assigner_puts for where it stands", node, module)
|
||||
}
|
||||
|
||||
// doRelease is the tool assigner_release.
|
||||
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||
key, err := delivery.PutKey(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept with the put")
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||
}
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
|
||||
}
|
||||
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
|
||||
data, err := delivery.EncodePut(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(putsState, key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
s.hear(key, data)
|
||||
return putAnswer{Node: node, Module: module, Put: p}, nil
|
||||
}
|
||||
|
||||
// doPuts is the tool assigner_puts.
|
||||
func (s *putTools) doPuts(args map[string]any) (any, error) {
|
||||
keys, err := s.st.keys(putsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the puts cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []putAnswer{}
|
||||
for _, key := range keys {
|
||||
node, module, err := delivery.ParsePutKey(key)
|
||||
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(putsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
p, err := delivery.DecodePut(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, putAnswer{Node: node, Module: module, Put: p})
|
||||
}
|
||||
return map[string]any{"puts": out}, nil
|
||||
}
|
||||
|
||||
func arg(args map[string]any, name string) string {
|
||||
switch v := args[name].(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(v)
|
||||
case bool:
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// The tools' names, as the manifest lists them.
|
||||
const (
|
||||
toolPut = "assigner_put"
|
||||
toolRelease = "assigner_release"
|
||||
toolPuts = "assigner_puts"
|
||||
)
|
||||
|
||||
// schema is a tool's arguments, each a string: required ones first named, and switches true or false.
|
||||
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
|
||||
props := map[string]any{}
|
||||
for name, description := range properties {
|
||||
p := map[string]any{"type": "string", "description": description}
|
||||
if slices.Contains(switches, name) {
|
||||
p["enum"] = []string{"true", "false"}
|
||||
}
|
||||
props[name] = p
|
||||
}
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
out["required"] = required
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tools are the assigner's tools as the runtime serves them (ADR 0546 clause 1).
|
||||
func (s *putTools) tools() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: toolPut, Description: "Put one version of a module on one machine (novox/hq ADR 0546): the build at that fingerprint, " +
|
||||
"the composition and grant the controller wrote for that machine, then the machine's want naming it, held there until " +
|
||||
"released when hold is true. A module whose moves a person takes (the bus, a module of policy record) is asked of the " +
|
||||
"operator first, and the bus has its streams snapshotted after the yes. Answers what was done or what it waits on; the " +
|
||||
"put goes on after the answer, each step said as the event mesh-delivery-assigner.put, and assigner_puts says where it stands.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine, where the module is assigned",
|
||||
"module": "the module",
|
||||
"version": "newest, a fingerprint (src2: and 64 hex digits), or at least its first 12 hex digits",
|
||||
"hold": "\"true\": the want stays at this version until released; \"false\" (the default): the rules move it again once it runs",
|
||||
"why": "why, kept with the put and said in any ask",
|
||||
}, []string{"node", "module", "version", "why"}, "hold"),
|
||||
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
|
||||
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
|
||||
[]string{"node", "module", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
|
||||
"its time and words. Narrowed by machine or module. Read-only.",
|
||||
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
|
||||
Run: s.doPuts},
|
||||
}
|
||||
}
|
||||
|
||||
// runtimeStore is the runtime's states.
|
||||
type runtimeStore struct{}
|
||||
|
||||
func (runtimeStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := stdio.State(state).Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (runtimeStore) put(state, key string, value []byte) error {
|
||||
_, err := stdio.State(state).Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
|
||||
func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||
|
||||
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
|
||||
func serveTools() error {
|
||||
s := newPutTools(runtimeStore{})
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
err := stdio.State(putsState).Watch("", func(c stdio.StateChange) error {
|
||||
s.hear(c.Key, c.Value)
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the puts are not heard yet (%v); asked again in %s\n", err, wait)
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
return stdio.Serve("", s.tools())
|
||||
}
|
||||
|
||||
// shortFP is a fingerprint's first twelve hex digits.
|
||||
func shortFP(fp string) string {
|
||||
part, err := delivery.FingerprintKeyPart(fp)
|
||||
if err != nil || len(part) < 12 {
|
||||
return fp
|
||||
}
|
||||
return part[:12]
|
||||
}
|
||||
@@ -326,13 +326,6 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
return n
|
||||
}
|
||||
|
||||
// value is what a bucket holds at a key now, nil when nothing.
|
||||
func (w *world) value(bucket, key string) []byte {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.values[bucket][key]
|
||||
}
|
||||
|
||||
// has says whether a bucket holds a key now.
|
||||
func (w *world) has(bucket, key string) bool {
|
||||
w.mu.Lock()
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -10,17 +10,14 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
|
||||
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
|
||||
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
||||
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
||||
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, asking),
|
||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
||||
// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
|
||||
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, 0),
|
||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
||||
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
||||
"lets the wants move to it now.", module, node, short(fp), words, module),
|
||||
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
||||
"no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
|
||||
Options: []asks.Option{
|
||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||
@@ -81,35 +78,6 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
|
||||
return c
|
||||
}
|
||||
|
||||
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||
// node and module, and cleared (raised false) when the node has a want for it again.
|
||||
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
|
||||
key := delivery.AssignerModule + ".neither." + node + "." + module
|
||||
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||
if !raised {
|
||||
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
|
||||
}
|
||||
headline := fmt.Sprintf("%s on %s is on no list", module, node)
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
|
||||
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||
}
|
||||
|
||||
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||
func LiftedCondition(module, fp string) asks.Condition {
|
||||
c := HeldCondition(module, fp, "", "", false, false)
|
||||
c.Resolved = LiftedWords(module, fp) + "."
|
||||
return c
|
||||
}
|
||||
|
||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||
const (
|
||||
OptionYes = "yes"
|
||||
|
||||
@@ -17,25 +17,13 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now()
|
||||
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
||||
a := HoldAsk(key, "mail", fNew, now)
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("the router refuses the ask: %v", err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||
t.Errorf("the ask cannot be written: %v", err)
|
||||
}
|
||||
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
||||
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
|
||||
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
|
||||
}
|
||||
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
||||
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
||||
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
|
||||
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
|
||||
t.Errorf("the ask asked again cannot be written: %v", err)
|
||||
}
|
||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||
t.Errorf("the view may not answer it: %v", err)
|
||||
}
|
||||
@@ -86,17 +74,6 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||
c := LiftedCondition("mail", fNew)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||
// the view, and name the move and the node.
|
||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
@@ -146,58 +123,4 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("the backup condition is refused: %v", err)
|
||||
}
|
||||
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||
// so the view cannot answer it, and it lasts no longer than an approving ask may.
|
||||
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
for _, bus := range []bool{false, true} {
|
||||
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
|
||||
if err != nil {
|
||||
t.Fatalf("bus %v: %v", bus, err)
|
||||
}
|
||||
if err := a.Ask.Check(now); err != nil {
|
||||
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
|
||||
}
|
||||
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
|
||||
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
|
||||
}
|
||||
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
|
||||
yes := a.Ask.Options[0]
|
||||
switch {
|
||||
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
|
||||
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
|
||||
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
|
||||
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
|
||||
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
|
||||
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
|
||||
}
|
||||
later := put
|
||||
later.At = now.Add(time.Minute)
|
||||
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
|
||||
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
|
||||
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
|
||||
}
|
||||
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||
t.Errorf("bus: the yes performs another composition's act")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||
// under the assigner's own name keyed by node and module, and is cleared under the same key.
|
||||
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
|
||||
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Fatalf("the condition is refused: %v", err)
|
||||
}
|
||||
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
|
||||
t.Errorf("condition = %+v", c)
|
||||
}
|
||||
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||
t.Errorf("cleared = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,252 +0,0 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
|
||||
// key; each pass takes it one step further from what the world holds (StepPut), and the rules pass over the node's want for
|
||||
// the module from its asking until it ends (Module.Puts).
|
||||
|
||||
// ComposeWithin is how long a put waits for the composition and grant it needs (a take-in of a build just done writes them)
|
||||
// before it is refused.
|
||||
const ComposeWithin = 10 * time.Minute
|
||||
|
||||
// PutWorld is what one step of a put reads.
|
||||
type PutWorld struct {
|
||||
Node, Module string
|
||||
Put delivery.Put
|
||||
// Assigned says the module stands assigned to the node.
|
||||
Assigned bool
|
||||
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
|
||||
Bus, Recorded bool
|
||||
// Build is the state of the builder's build at the put's fingerprint, "" when no key is written; BuildWords its error.
|
||||
Build, BuildWords string
|
||||
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||
Newest string
|
||||
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||
Composed *delivery.ComposedDigest
|
||||
Granted bool
|
||||
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
|
||||
// that ask expired unanswered.
|
||||
Answer string
|
||||
AskExpired bool
|
||||
// Snapshot says where the bus's streams were snapshotted for this put, once they were; Snapshotting that it was
|
||||
// asked; SnapshotFailed why it was not taken.
|
||||
Snapshot string
|
||||
Snapshotting bool
|
||||
SnapshotFailed string
|
||||
// Have is what the node says it runs of the module; nil when it says nothing.
|
||||
Have *delivery.Have
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
|
||||
type PutNext struct {
|
||||
Put delivery.Put
|
||||
// Ask writes the put's ask about the move (PutAsk).
|
||||
Ask bool
|
||||
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
|
||||
Snapshot bool
|
||||
// Warranted records, once, in the controller's hand-act log that the bus's want was written on the operator's warrant.
|
||||
Warranted bool
|
||||
}
|
||||
|
||||
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
|
||||
func StepPut(w PutWorld) PutNext {
|
||||
p := w.Put
|
||||
if delivery.PutEnded(p.Step) {
|
||||
return PutNext{Put: p}
|
||||
}
|
||||
say := func(step, format string, args ...any) PutNext {
|
||||
return PutNext{Put: p.Said(w.Now, step, fmt.Sprintf(format, args...))}
|
||||
}
|
||||
fp := short(p.FP)
|
||||
if !w.Assigned {
|
||||
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||
}
|
||||
switch w.Build {
|
||||
case delivery.BuildDone:
|
||||
case delivery.BuildFailed:
|
||||
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
|
||||
case "":
|
||||
if p.FP != w.Newest {
|
||||
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
|
||||
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
|
||||
}
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
|
||||
default:
|
||||
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||
}
|
||||
if w.Composed == nil || !w.Granted {
|
||||
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||
if w.Composed != nil {
|
||||
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
|
||||
}
|
||||
since := w.Now
|
||||
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
|
||||
since = p.Steps[len(p.Steps)-1].At
|
||||
}
|
||||
if w.Now.Sub(since) > ComposeWithin {
|
||||
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
|
||||
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
|
||||
}
|
||||
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
|
||||
}
|
||||
d := *w.Composed
|
||||
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
|
||||
return PutNext{Put: applied(w, p)}
|
||||
}
|
||||
p.Composed = d.Digest
|
||||
if needsPerson(w, d) {
|
||||
switch w.Answer {
|
||||
case OptionNo:
|
||||
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
|
||||
case OptionYes:
|
||||
default:
|
||||
if w.AskExpired {
|
||||
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
|
||||
}
|
||||
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
|
||||
if p.Ask == "" {
|
||||
next.Ask = true
|
||||
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
|
||||
}
|
||||
return next
|
||||
}
|
||||
if w.Bus {
|
||||
switch {
|
||||
case w.Snapshot != "":
|
||||
p.Snapshot = w.Snapshot
|
||||
case w.SnapshotFailed != "":
|
||||
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
|
||||
case w.Snapshotting:
|
||||
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
default:
|
||||
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
|
||||
next.Snapshot = true
|
||||
return next
|
||||
}
|
||||
}
|
||||
}
|
||||
words := fmt.Sprintf("the want of %s on %s names %s (composition %s)", w.Module, w.Node, fp, short(d.Digest))
|
||||
if p.Hold {
|
||||
words += ", held until released"
|
||||
}
|
||||
if p.Snapshot != "" {
|
||||
words += "; the streams were snapshotted: " + p.Snapshot
|
||||
}
|
||||
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
|
||||
next.Warranted = w.Bus && p.Ask != ""
|
||||
return next
|
||||
}
|
||||
|
||||
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
|
||||
func applied(w PutWorld, p delivery.Put) delivery.Put {
|
||||
h := w.Have
|
||||
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
|
||||
return p
|
||||
}
|
||||
switch {
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
|
||||
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
|
||||
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
|
||||
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
|
||||
case failed(*h):
|
||||
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
|
||||
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
|
||||
// container, move data, or that cannot be judged (§14).
|
||||
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
|
||||
runs := w.Have != nil && w.Have.FP == w.Put.FP
|
||||
if runs && w.Have.Composed == d.Digest {
|
||||
return false // the node runs this composition already: nothing moves
|
||||
}
|
||||
if w.Bus {
|
||||
return !(runs && d.UserListOnly)
|
||||
}
|
||||
if !w.Recorded {
|
||||
return false
|
||||
}
|
||||
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
|
||||
}
|
||||
|
||||
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
|
||||
func PutWant(p delivery.Put) delivery.Want {
|
||||
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||
}
|
||||
|
||||
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
|
||||
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
|
||||
switch p.Step {
|
||||
case delivery.PutWanted, delivery.PutHolding:
|
||||
w := PutWant(p)
|
||||
return &w, true
|
||||
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
|
||||
return nil, false
|
||||
}
|
||||
return nil, true
|
||||
}
|
||||
|
||||
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
|
||||
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
|
||||
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
|
||||
func PutAsking(p delivery.Put) int {
|
||||
if n := p.At.UnixMilli(); n > 0 {
|
||||
return int(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
|
||||
func PutAct(node, module, fp, composed string) asks.Act {
|
||||
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
|
||||
}
|
||||
|
||||
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
|
||||
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
|
||||
// channel that proves who answered takes it (ADR 0259), never the view.
|
||||
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
|
||||
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
|
||||
what := map[delivery.Judgement]string{
|
||||
delivery.JudgedRecreates: "recreates a container",
|
||||
delivery.JudgedMovesData: "moves the module's data",
|
||||
delivery.JudgedCannot: "cannot be judged: " + d.Words,
|
||||
delivery.JudgedNeither: "recreates no container and moves no data",
|
||||
delivery.JudgedUnchanged: "changes nothing the node runs",
|
||||
}[d.Judgement]
|
||||
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
|
||||
expires := now.Add(7 * 24 * time.Hour)
|
||||
if bus {
|
||||
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, err
|
||||
}
|
||||
yes.Level, yes.Binds = asks.Approve, binds
|
||||
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
|
||||
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
|
||||
expires = now.Add(asks.ApproveLasts)
|
||||
}
|
||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||
ID: delivery.AskingID(key, PutAsking(p)),
|
||||
Headline: fmt.Sprintf("Put %s on %s?", module, node),
|
||||
Explanation: explanation,
|
||||
Options: []asks.Option{yes,
|
||||
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
|
||||
Who: asks.Operator,
|
||||
Expires: expires,
|
||||
OnExpiry: "the node keeps what it runs",
|
||||
About: key,
|
||||
}, Asking: PutAsking(p)}, nil
|
||||
}
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -15,20 +13,13 @@ import (
|
||||
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
||||
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
||||
type Module struct {
|
||||
Name string
|
||||
FP string
|
||||
Built bool
|
||||
Held bool
|
||||
Granted bool
|
||||
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
|
||||
// that runs an older one names it (issue 1945). Nil: none.
|
||||
GrantedAt func(fp string) bool
|
||||
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
|
||||
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
|
||||
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||
GrantedOn func(node string) bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
Name string
|
||||
FP string
|
||||
Built bool
|
||||
Held bool
|
||||
Granted bool
|
||||
Manifest catalogue.Manifest
|
||||
Nodes []string
|
||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||
@@ -40,12 +31,6 @@ type Module struct {
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
||||
Restored func(node string) bool
|
||||
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
|
||||
Kept bool
|
||||
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
Puts map[string]delivery.Put
|
||||
}
|
||||
|
||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
@@ -84,40 +69,27 @@ func Canary(nodes []string, online map[string]bool) string {
|
||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||
// limit (ADR 0528 clause 3).
|
||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||
return holds(m, node, n, true)
|
||||
}
|
||||
|
||||
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
|
||||
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
|
||||
// composition.
|
||||
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
switch {
|
||||
case !m.Built:
|
||||
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
|
||||
case m.Held:
|
||||
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
||||
}
|
||||
if ordering {
|
||||
canary := Canary(m.Nodes, n.Online)
|
||||
switch {
|
||||
case canary == "":
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
canary := Canary(m.Nodes, n.Online)
|
||||
switch {
|
||||
case canary == "":
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||
}
|
||||
for _, provision := range m.Manifest.Requires {
|
||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||
return false, delivery.RuleProviders, why
|
||||
}
|
||||
}
|
||||
if !m.Granted {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||
}
|
||||
if m.GrantedOn != nil && !m.GrantedOn(node) {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
||||
short(m.FP))
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||
}
|
||||
@@ -166,9 +138,6 @@ type Outcome struct {
|
||||
Wants map[string]delivery.Want
|
||||
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||
Held *delivery.Held
|
||||
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||
Lifted bool
|
||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||
Record []RecordNeed
|
||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||
@@ -176,11 +145,6 @@ type Outcome struct {
|
||||
Restore []string
|
||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||
Rules map[string]Holding
|
||||
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||
// own composition and is never here.
|
||||
Neither map[string]Holding
|
||||
}
|
||||
|
||||
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
||||
@@ -204,15 +168,12 @@ type Holding struct {
|
||||
// module holding data is not set back but asked about (Restore). A
|
||||
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
|
||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
||||
held := m.Held
|
||||
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
|
||||
out.Lifted, held = true, false
|
||||
}
|
||||
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||
if !held && !m.Released && m.FP != "" {
|
||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
|
||||
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
||||
out.Held = &delivery.Held{Node: node, Words: h.Detail}
|
||||
held = true
|
||||
break
|
||||
}
|
||||
@@ -226,29 +187,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
before, had := wants[node]
|
||||
var next delivery.Want
|
||||
p, put := m.Puts[node]
|
||||
if !put && had && before.Put && before.Hold {
|
||||
// The want is the truth (ADR 0546 clause 5): a want marked put and held stands while its put is not heard (the
|
||||
// puts not open, or not yet handed after a start); only a put heard released gives it back to the rules. A put
|
||||
// without hold is not kept this way: its put takes it further once heard.
|
||||
p, put = delivery.Put{FP: before.FP, Composed: before.Composed, Hold: true, Why: strings.TrimPrefix(before.Words, "put: "),
|
||||
Step: delivery.PutHolding}, true
|
||||
}
|
||||
if put {
|
||||
if want, owns := PutOwns(p); owns {
|
||||
if want != nil {
|
||||
w := *want
|
||||
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
|
||||
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
|
||||
w.Composed = ul.Composed
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||
@@ -258,49 +196,15 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
}
|
||||
continue
|
||||
}
|
||||
ok, rule, words := Holds(mm, node, n)
|
||||
target := m.FP
|
||||
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
|
||||
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
|
||||
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
|
||||
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
|
||||
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
|
||||
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
|
||||
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
|
||||
// order the move from it as from any want.
|
||||
switch _, composed := m.composition(node, have.FP); {
|
||||
case !ran:
|
||||
case have.FP == m.FP:
|
||||
ok, rule, words = holds(mm, node, n, false)
|
||||
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
|
||||
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
|
||||
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
|
||||
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
|
||||
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
|
||||
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
|
||||
ok, target = true, have.FP
|
||||
}
|
||||
}
|
||||
switch {
|
||||
switch ok, rule, words := Holds(mm, node, n); {
|
||||
case ok:
|
||||
d, _ := m.composition(node, target)
|
||||
next = delivery.Want{FP: target, Composed: d.Digest}
|
||||
at := m
|
||||
if target != m.FP {
|
||||
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
|
||||
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||
at.FP = target
|
||||
}
|
||||
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||
d, _ := m.composition(node, m.FP)
|
||||
next = delivery.Want{FP: m.FP, Composed: d.Digest}
|
||||
if had && !before.Removed && before.FP == m.FP && before.Composed == d.Digest {
|
||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||
}
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
// A removed want is no want: a first want after it goes through the record ask as a first want does.
|
||||
asked, hadLive := before, had
|
||||
if before.Removed {
|
||||
asked, hadLive = delivery.Want{}, false
|
||||
}
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need, wait := m.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
@@ -329,43 +233,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
default:
|
||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||
if !had || before.Removed {
|
||||
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
|
||||
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||
h, there := n.Haves[node][m.Name]
|
||||
switch {
|
||||
case there && h.Phase != delivery.PhaseRemoved && !held:
|
||||
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||
case had && (!there || h.Phase == delivery.PhaseRemoved):
|
||||
// A removed want of a node that does not run the module says why no want follows it (issue 1945), so the view's
|
||||
// assigner_want answers it; it stays removed. A node with no key gets none: a removed want written where none was
|
||||
// would have its node agent remove the module and stop holding its composition's resources back.
|
||||
next = delivery.Want{Removed: true, Words: NoWantWords(rule, words)}
|
||||
if !delivery.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
|
||||
if before.FP == m.FP {
|
||||
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
|
||||
next = delivery.Want{FP: m.FP, Composed: before.Composed}
|
||||
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
|
||||
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
|
||||
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
|
||||
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
|
||||
at := m
|
||||
at.FP = before.FP
|
||||
running, ran := n.Haves[node][m.Name]
|
||||
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
}
|
||||
if !wait {
|
||||
next.Composed = d.Digest
|
||||
}
|
||||
}
|
||||
}
|
||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||
@@ -380,63 +252,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
|
||||
func NoWantWords(rule int, words string) string {
|
||||
return fmt.Sprintf("no want: rule %d: %s", rule, words)
|
||||
}
|
||||
|
||||
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||
// moving). A failed health while it
|
||||
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
|
||||
// the world's lastHealthy) already asks for phase running.
|
||||
func failed(h delivery.Have) bool {
|
||||
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||
}
|
||||
|
||||
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||
func runs(h delivery.Have) bool {
|
||||
return h.Phase == delivery.PhaseRunning
|
||||
}
|
||||
|
||||
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
|
||||
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
|
||||
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
|
||||
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
|
||||
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
|
||||
func allRunHealthy(m Module, n Nodes) bool {
|
||||
ran := 0
|
||||
for _, node := range m.Nodes {
|
||||
h, ok := n.Haves[node][m.Name]
|
||||
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
|
||||
continue
|
||||
}
|
||||
if !healthy(h, m.FP) {
|
||||
return false
|
||||
}
|
||||
ran++
|
||||
}
|
||||
return ran > 0
|
||||
}
|
||||
|
||||
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
|
||||
// it, and the node's own words. The node is the held record's own field.
|
||||
func HeldWords(h delivery.Have, at time.Time) string {
|
||||
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
|
||||
if !at.IsZero() {
|
||||
words += ", seen at " + at.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if d := strings.TrimSpace(h.Detail); d != "" {
|
||||
return words + ": " + d
|
||||
}
|
||||
return words + ": the node gave no words"
|
||||
}
|
||||
|
||||
// LiftedWords say why a hold was lifted by itself.
|
||||
func LiftedWords(module, fp string) string {
|
||||
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
|
||||
}
|
||||
|
||||
// normal is a want without its version, as a decoded and a composed one are compared.
|
||||
func normal(w delivery.Want) delivery.Want {
|
||||
w.V = 0
|
||||
@@ -514,9 +329,3 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
|
||||
}
|
||||
return w, true
|
||||
}
|
||||
|
||||
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
|
||||
// put reads by too).
|
||||
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||
return g.NamesNode(node, module)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -86,33 +85,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
||||
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
||||
m, n := ready()
|
||||
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["ace"]; w.FP != fNew {
|
||||
t.Errorf("ace, whose account the grant names, wants %+v", w)
|
||||
}
|
||||
if got := out.Rules["g14"]; got.Rule != delivery.RuleGrant || !strings.Contains(got.Words, "on g14") {
|
||||
t.Errorf("g14, whose account the grant does not name, is held by %+v", got)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
accounts []string
|
||||
node, module string
|
||||
named bool
|
||||
}{
|
||||
{nil, "g14", "mail", true},
|
||||
{[]string{"g14.mail.send"}, "g14", "mail", true},
|
||||
{[]string{"g14.mailer"}, "g14", "mail", false},
|
||||
{[]string{"ace.mail"}, "g14", "mail", false},
|
||||
} {
|
||||
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
|
||||
@@ -138,10 +110,8 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
|
||||
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
||||
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
||||
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
|
||||
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
|
||||
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
|
||||
t.Fatalf("held = %+v", out.Held)
|
||||
}
|
||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||
@@ -248,25 +218,6 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
|
||||
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
|
||||
t.Errorf("a revision was written while a rule holds: %+v", w)
|
||||
}
|
||||
|
||||
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
|
||||
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
|
||||
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
|
||||
out = Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["g14"]; w.FP != fOld || w.Composed != revised || w.Newest != fNew {
|
||||
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
|
||||
}
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord,
|
||||
Against: dig(fOld), Words: "recreates"}, true
|
||||
}
|
||||
out = Wants(m, n, wantedOld(), nil)
|
||||
if w := out.Wants["g14"]; w.Composed == revised {
|
||||
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
|
||||
}
|
||||
if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) {
|
||||
t.Errorf("the revision was not asked: %+v", out.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
|
||||
@@ -350,7 +301,6 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
|
||||
}
|
||||
m, n, _ = movingTo(delivery.JudgedRecreates)
|
||||
recording(&m)
|
||||
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
|
||||
delete(n.Haves["ace"], "mail")
|
||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||
t.Errorf("a node that never ran it: %+v", out)
|
||||
@@ -384,142 +334,6 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
if len(out.Record) != 0 {
|
||||
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
||||
}
|
||||
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
|
||||
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
|
||||
put := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
step string
|
||||
wanted map[string]delivery.Want
|
||||
want *delivery.Want // nil: nothing written on ace
|
||||
}{
|
||||
{"waiting for the operator", delivery.PutPerson, wantedOld(), nil},
|
||||
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||
{"not heard: a held want at an older fingerprint stands", "", map[string]delivery.Want{"ace": {FP: fOld, Composed: dig(fOld), Put: true, Hold: true, Words: "put: back"}}, nil},
|
||||
{"not heard: a held want the rules would rewrite stands", "", map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||
} {
|
||||
p := put
|
||||
p.Step = c.step
|
||||
m.Puts = map[string]delivery.Put{"ace": p}
|
||||
if c.step == "" {
|
||||
m.Puts = nil
|
||||
}
|
||||
out := Wants(m, n, c.wanted, nil)
|
||||
got, wrote := out.Wants["ace"]
|
||||
switch {
|
||||
case c.want == nil && wrote:
|
||||
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
|
||||
case c.want != nil && (!wrote || !delivery.SameWant(normal(got), *c.want)):
|
||||
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
|
||||
}
|
||||
}
|
||||
userList := "sha256:" + strings.Repeat("7", 64)
|
||||
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
held := put
|
||||
held.Step = delivery.PutHolding
|
||||
m.Puts = map[string]delivery.Put{"ace": held}
|
||||
if w := Wants(m, n, map[string]delivery.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
|
||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||
}
|
||||
m.Puts = nil
|
||||
stepsOfAPut(t)
|
||||
}
|
||||
|
||||
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
|
||||
// operator's yes, the bus's snapshot, the want, and the have.
|
||||
func stepsOfAPut(t *testing.T) {
|
||||
t.Helper()
|
||||
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
|
||||
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
bus := func() PutWorld {
|
||||
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
|
||||
Composed: &d, Granted: true, Have: &running, Now: now}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
world func() PutWorld
|
||||
step string
|
||||
says string
|
||||
ask, snap bool
|
||||
warranted bool
|
||||
}{
|
||||
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, delivery.PutRefused, "not assigned", false, false, false},
|
||||
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, delivery.PutBuild, "builds by itself", false, false, false},
|
||||
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, delivery.PutRefused, "holds no build", false, false, false},
|
||||
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = delivery.BuildFailed, "go vet"; return w }, delivery.PutRefused, "go vet", false, false, false},
|
||||
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, delivery.PutCompose, "no composition of nats for novox", false, false, false},
|
||||
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, delivery.PutCompose, "does not name novox", false, false, false},
|
||||
{"none within the bound: refused", func() PutWorld {
|
||||
w := bus()
|
||||
w.Composed = nil
|
||||
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), delivery.PutCompose, "waits")
|
||||
return w
|
||||
}, delivery.PutRefused, "mesh-hand-compose", false, false, false},
|
||||
{"the bus asks the operator", bus, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, delivery.PutRefused, "answered no", false, false, false},
|
||||
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, delivery.PutRefused, "expired unanswered", false, false, false},
|
||||
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, delivery.PutSnapshot, "asked novox's backup holder", false, true, false},
|
||||
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, delivery.PutSnapshot, "waits for novox's backup holder", false, false, false},
|
||||
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, delivery.PutRefused, "no restic", false, false, false},
|
||||
{"snapshotted: the want, warranted", func() PutWorld {
|
||||
w := bus()
|
||||
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
|
||||
w.Put.Ask, _ = delivery.RecordAskKey("novox", "nats", d.Digest)
|
||||
return w
|
||||
}, delivery.PutWanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
|
||||
{"a user-list revision of the bus needs nobody", func() PutWorld {
|
||||
w := bus()
|
||||
ul := d
|
||||
ul.UserListOnly = true
|
||||
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutWanted, "held until released", false, false, false},
|
||||
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
|
||||
{"a recorded module judged to recreate asks", func() PutWorld {
|
||||
w := bus()
|
||||
w.Bus, w.Recorded = false, true
|
||||
r := d
|
||||
r.Judgement = delivery.JudgedRecreates
|
||||
w.Composed = &r
|
||||
return w
|
||||
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||
{"wanted, the node runs it: holding", func() PutWorld {
|
||||
w := bus()
|
||||
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||
w.Put.Composed = d.Digest
|
||||
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutHolding, "held there until released", false, false, false},
|
||||
{"wanted without hold: done", func() PutWorld {
|
||||
w := bus()
|
||||
w.Put.Hold = false
|
||||
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||
w.Put.Composed = d.Digest
|
||||
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return w
|
||||
}, delivery.PutDone, "the rules write its want again", false, false, false},
|
||||
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
next := StepPut(c.world())
|
||||
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
||||
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
||||
}
|
||||
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
|
||||
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
|
||||
c.ask, c.snap, c.warranted)
|
||||
}
|
||||
if next.Ask && next.Put.Ask == "" {
|
||||
t.Errorf("put, %s: an ask without its key", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
||||
@@ -706,370 +520,3 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
|
||||
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
|
||||
m, n := ready()
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
|
||||
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
|
||||
t.Fatalf("held = %+v", out.Held)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
|
||||
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
|
||||
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
|
||||
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||
everywhere := func() (Module, Nodes) {
|
||||
m, n := ready()
|
||||
m.Held = true
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
return m, n
|
||||
}
|
||||
m, n := everywhere()
|
||||
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
|
||||
if !out.Lifted || out.Held != nil {
|
||||
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
|
||||
}
|
||||
for _, node := range []string{"ace", "g14"} {
|
||||
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
|
||||
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
|
||||
}
|
||||
}
|
||||
|
||||
m, n = everywhere()
|
||||
delete(n.Haves["g14"], "mail")
|
||||
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
|
||||
t.Error("a node that does not run the module kept the hold")
|
||||
}
|
||||
|
||||
for name, change := range map[string]func(*Module, *Nodes){
|
||||
"set back": func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
},
|
||||
"failing": func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
},
|
||||
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
|
||||
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
|
||||
"not held": func(m *Module, _ *Nodes) { m.Held = false },
|
||||
} {
|
||||
m, n := everywhere()
|
||||
change(&m, &n)
|
||||
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
|
||||
t.Errorf("%s: the hold was lifted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld
|
||||
// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld
|
||||
// (health unknown) and holds no want for it. The canary g14 does not run F.
|
||||
func x11() (Module, Nodes) {
|
||||
m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"},
|
||||
GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld },
|
||||
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
|
||||
n := Nodes{
|
||||
Online: map[string]bool{"g14": true, "shanks": true},
|
||||
Haves: map[string]map[string]delivery.Have{
|
||||
"g14": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
||||
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
||||
"shanks": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
||||
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
||||
},
|
||||
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
|
||||
}
|
||||
return m, n
|
||||
}
|
||||
|
||||
// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by
|
||||
// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that
|
||||
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
|
||||
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
|
||||
m, n := x11()
|
||||
out := Wants(m, n, nil, nil)
|
||||
for _, node := range []string{"g14", "shanks"} {
|
||||
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
|
||||
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
|
||||
}
|
||||
}
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("left on neither list: %+v", out.Neither)
|
||||
}
|
||||
// with the provider healthy, only the canary holds shanks back: still at what it runs
|
||||
m, n = x11()
|
||||
n.Haves["g14"]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
|
||||
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.FP != fOld || w.Rule != delivery.RuleCanary {
|
||||
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
|
||||
}
|
||||
// a node already running F gets its want at F; a removed want is no want either
|
||||
m, n = x11()
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
out = Wants(m, n, map[string]delivery.Want{"g14": {Removed: true}}, nil)
|
||||
if w := out.Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) || w.Removed {
|
||||
t.Errorf("g14 running F with a removed want = %+v, want F", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no
|
||||
// want past the rules, and is named on neither list; the ordering rules still order every move from a want.
|
||||
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||
for what, change := range map[string]func(*Module, *Nodes){
|
||||
"an old-form build": func(_ *Module, n *Nodes) {
|
||||
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
},
|
||||
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
|
||||
"no composition": func(m *Module, _ *Nodes) {
|
||||
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, fp == fNew
|
||||
}
|
||||
},
|
||||
} {
|
||||
m, n := x11()
|
||||
change(&m, &n)
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
|
||||
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
|
||||
}
|
||||
}
|
||||
m, n := x11()
|
||||
before := map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}, "shanks": {FP: fOld, Composed: dig(fOld)}}
|
||||
out := Wants(m, n, before, nil)
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
|
||||
}
|
||||
for node, w := range out.Wants {
|
||||
if w.FP != fOld {
|
||||
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is
|
||||
// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new
|
||||
// form goes with the node's own composition and is never named.
|
||||
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||
m, n := x11()
|
||||
delete(n.Haves["shanks"], "screen-lock")
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
|
||||
}
|
||||
if _, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
|
||||
}
|
||||
m.Built = false
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
||||
}
|
||||
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
|
||||
// the module has the reason written into its removed want, once; a node with no key gets no want written, and a node that
|
||||
// runs the module is on neither list, its removed want left as it is.
|
||||
gone := delivery.Want{Removed: true, Words: "screen-lock is no longer assigned to shanks"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
before map[string]delivery.Want
|
||||
want map[string]delivery.Want
|
||||
}{
|
||||
{"no key: nothing written", nil, map[string]delivery.Want{}},
|
||||
{"a removed want: it says why", map[string]delivery.Want{"shanks": gone},
|
||||
map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
|
||||
{"a removed want that says why already: nothing written", map[string]delivery.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
|
||||
map[string]delivery.Want{}},
|
||||
{"a node that runs it: on neither list, its removed want left", map[string]delivery.Want{"g14": gone}, map[string]delivery.Want{}},
|
||||
} {
|
||||
out := Wants(m, n, c.before, nil)
|
||||
if len(out.Wants) != len(c.want) {
|
||||
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
|
||||
continue
|
||||
}
|
||||
for node, w := range c.want {
|
||||
if !delivery.SameWant(normal(out.Wants[node]), w) {
|
||||
t.Errorf("%s: want of %s = %+v, want %+v", c.name, node, out.Wants[node], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
m.FP = ""
|
||||
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
||||
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
|
||||
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
|
||||
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
|
||||
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
|
||||
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||
oldForm := func(n Nodes, node string) {
|
||||
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
}
|
||||
providersHealthy := func(n Nodes) {
|
||||
for _, node := range []string{"g14", "shanks"} {
|
||||
n.Haves[node]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
}
|
||||
}
|
||||
// the canary g14 runs F healthy: shanks's first want names F
|
||||
m, n := x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out := Wants(m, n, nil, nil)
|
||||
if w := out.Wants["shanks"]; w.FP != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
|
||||
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
|
||||
}
|
||||
if _, ok := out.Neither["shanks"]; ok {
|
||||
t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither)
|
||||
}
|
||||
// the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3
|
||||
m, n = x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
|
||||
}
|
||||
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
|
||||
}
|
||||
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
|
||||
m, n = x11()
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, nil, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("providers not healthy: shanks got a want: %+v", w)
|
||||
}
|
||||
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
|
||||
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
|
||||
}
|
||||
// F not granted: nothing, even with the canary healthy
|
||||
m, n = x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "shanks")
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
m.Granted = false
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
|
||||
t.Errorf("F not granted: shanks got a want: %+v", w)
|
||||
}
|
||||
// the old-form node is the canary itself: F once its providers run healthy
|
||||
m, n = x11()
|
||||
providersHealthy(n)
|
||||
oldForm(n, "g14")
|
||||
if w := Wants(m, n, nil, nil).Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||
t.Errorf("old-form canary: g14's want = %+v, want F", w)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking.
|
||||
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
|
||||
// canary two seconds before the new build was healthy.
|
||||
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
|
||||
for _, phase := range []delivery.Phase{delivery.PhaseBackingUp, delivery.PhaseApplying, delivery.PhaseChecking} {
|
||||
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
||||
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
||||
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
||||
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
||||
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
|
||||
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
|
||||
}
|
||||
}
|
||||
for _, phase := range []delivery.Phase{delivery.PhaseRunning, delivery.PhaseFailed} {
|
||||
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
||||
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
||||
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
||||
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
||||
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
|
||||
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module
|
||||
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
|
||||
// fingerprint, or failed, still does.
|
||||
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
|
||||
setup := func(phase delivery.Phase, fp string, health delivery.Health) (Module, Nodes) {
|
||||
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
|
||||
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]delivery.Have{
|
||||
"ace": {"web": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
||||
"g14": {"web": {FP: fp, Phase: phase, Health: health}}}}
|
||||
return m, n
|
||||
}
|
||||
for _, phase := range []delivery.Phase{delivery.PhaseRemoved, delivery.PhaseHeld, delivery.PhaseApplying, delivery.PhaseBackingUp, delivery.PhaseChecking} {
|
||||
m, n := setup(phase, fOld, delivery.HealthUnknown)
|
||||
if out := Wants(m, n, nil, nil); !out.Lifted {
|
||||
t.Errorf("g14's have in phase %s kept the hold up", phase)
|
||||
}
|
||||
}
|
||||
for what, h := range map[string]delivery.Have{
|
||||
"running another fingerprint": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||
"failed": {FP: fNew, Phase: delivery.PhaseFailed, Health: delivery.HealthFailed},
|
||||
} {
|
||||
m, n := setup(h.Phase, h.FP, h.Health)
|
||||
if out := Wants(m, n, nil, nil); out.Lifted {
|
||||
t.Errorf("g14 %s: the hold was lifted", what)
|
||||
}
|
||||
}
|
||||
// a failed health on a removed have holds nothing
|
||||
m, n := setup(delivery.PhaseRemoved, fNew, delivery.HealthFailed)
|
||||
m.Held = false
|
||||
if out := Wants(m, n, nil, nil); out.Held != nil {
|
||||
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
|
||||
}
|
||||
}
|
||||
|
||||
// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a
|
||||
// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes.
|
||||
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
|
||||
m, n := x11()
|
||||
recording(&m)
|
||||
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord}, true
|
||||
}
|
||||
out := Wants(m, n, map[string]delivery.Want{"shanks": {Removed: true}}, nil)
|
||||
if w, ok := out.Wants["shanks"]; ok {
|
||||
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
|
||||
}
|
||||
asked := false
|
||||
for _, r := range out.Record {
|
||||
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == delivery.JudgedRecreates)
|
||||
}
|
||||
if !asked {
|
||||
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the
|
||||
// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on
|
||||
// neither list: a held module is on the held list, whose hold is said and answered on its own.
|
||||
func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
|
||||
for what, change := range map[string]func(*Module, *Nodes){
|
||||
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
|
||||
"held at F, running an old-form build": func(m *Module, n *Nodes) {
|
||||
m.Held, m.Kept = true, true
|
||||
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
},
|
||||
"held by this pass": func(_ *Module, n *Nodes) {
|
||||
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||
},
|
||||
} {
|
||||
m, n := x11()
|
||||
change(&m, &n)
|
||||
out := Wants(m, n, nil, nil)
|
||||
if len(out.Neither) != 0 {
|
||||
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
|
||||
}
|
||||
if out.Rules["shanks"].Rule != delivery.RuleNotHeld {
|
||||
t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
|
||||
|
||||
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||
// the states it declares (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||
func TestManifest(t *testing.T) {
|
||||
m := load(t)
|
||||
if m.Module != "mesh-delivery-assigner" {
|
||||
@@ -42,7 +42,7 @@ func TestManifest(t *testing.T) {
|
||||
for _, s := range m.State {
|
||||
states[s.Name] = s.History
|
||||
}
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||
@@ -56,16 +56,7 @@ func TestManifest(t *testing.T) {
|
||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||
}
|
||||
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
|
||||
// calls the controller's warranted and the bus node's backup holder.
|
||||
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
|
||||
}
|
||||
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||
t.Errorf("emits = %v, want put", m.Emits)
|
||||
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
||||
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,24 +20,10 @@
|
||||
"node-build-go",
|
||||
"operator-channel"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.warranted",
|
||||
"seat:node-backup.now",
|
||||
"seat:node-backup.backed-up"
|
||||
],
|
||||
"emits": [
|
||||
"put"
|
||||
],
|
||||
"tools": [
|
||||
"assigner_put",
|
||||
"assigner_release",
|
||||
"assigner_puts"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "asks", "history": 5 },
|
||||
{ "name": "compositions", "history": 5 },
|
||||
{ "name": "differences", "history": 64 },
|
||||
{ "name": "puts", "history": 20 },
|
||||
{ "name": "wants", "history": 20 }
|
||||
],
|
||||
"own-secrets": {
|
||||
@@ -85,10 +71,7 @@
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-assigner",
|
||||
"binary": "mesh-delivery-assigner",
|
||||
"loads": [
|
||||
"mesh-delivery-assigner"
|
||||
]
|
||||
"binary": "mesh-delivery-assigner"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,178 +0,0 @@
|
||||
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
|
||||
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
|
||||
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
|
||||
// other program: it serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueAsker writes a queues value and waits for the builder's answer to it.
|
||||
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: build.StuckTool,
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
||||
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
||||
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
||||
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
||||
{Name: build.AskAgainTool,
|
||||
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
|
||||
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
|
||||
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
|
||||
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := askAgain(key, value); err != nil {
|
||||
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
|
||||
}
|
||||
return map[string]any{"key": key, "asked": true,
|
||||
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
||||
}},
|
||||
{Name: build.SeatQueueTool,
|
||||
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
||||
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
||||
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
|
||||
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
|
||||
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
|
||||
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
|
||||
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
language := build.LanguageOf(str(args, "language"))
|
||||
if language == "" {
|
||||
language = build.LanguageOf(str(args, "seat"))
|
||||
}
|
||||
if language == "" {
|
||||
language = delivery.AskGo
|
||||
}
|
||||
limit, _ := args["limit"].(float64)
|
||||
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return queue(key, value)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
||||
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
||||
return func(key string, value []byte) (json.RawMessage, error) {
|
||||
answered := make(chan json.RawMessage, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
||||
return nil
|
||||
}
|
||||
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
|
||||
select {
|
||||
case answered <- c.Value:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
// The watch may have heard the answer before the revision was known: read it once too.
|
||||
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
|
||||
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
|
||||
return e.Value, nil
|
||||
}
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case v := <-answered:
|
||||
return v, nil
|
||||
case <-wait.Done():
|
||||
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) build.StateRead {
|
||||
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
return e.Value, time.Time{}, nil
|
||||
}, Written: func(key string) time.Time {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return written(s, key, e.Revision)
|
||||
}}
|
||||
}
|
||||
|
||||
// written is when the key's revision was written, from its history, or zero.
|
||||
func written(s stdio.KeptState, key string, revision uint64) time.Time {
|
||||
h, err := s.History(key)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, r := range h {
|
||||
if r.Revision == revision {
|
||||
return r.At
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func main() {
|
||||
steps := map[string]build.StateRead{}
|
||||
for _, bm := range build.StuckBuildModules {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
return err
|
||||
}
|
||||
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -66,11 +66,6 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return settled(ctx, w, each)
|
||||
}
|
||||
|
||||
// settled hands each entry the watch delivers before its end-of-initial marker to each, and answers the watch.
|
||||
func settled(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
||||
@@ -4,303 +4,73 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
||||
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
|
||||
// be answered (issue 1945). Never polled.
|
||||
// (issue 1910): read once, at the take-over of a key at start, never polled.
|
||||
type heldOnTheBus struct{ js jetstream.JetStream }
|
||||
|
||||
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
||||
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
||||
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
|
||||
seen, err := h.seen(ctx, a, false)
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return seen.Step || seen.OnStream, nil
|
||||
}
|
||||
|
||||
// Gone is build.Builder's Gone: why the ask can no longer be answered (build.AskSeen.Why), or "".
|
||||
func (h heldOnTheBus) Gone(ctx context.Context, a delivery.StepAsk) (string, error) {
|
||||
seen, err := h.seen(ctx, a, true)
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return seen.Why(), nil
|
||||
}
|
||||
|
||||
// maxOnSubject bounds the messages of one subject a read walks (each a read by sequence, next by subject), and maxSteps
|
||||
// the steps it reads whole to find the last ask taken: past them the read says it cannot tell, and the ask is waited for.
|
||||
const (
|
||||
maxOnSubject = 5000
|
||||
maxSteps = 200
|
||||
)
|
||||
|
||||
// seen reads what the bus holds of the ask; with later, the ask asked last that a build node took too (AskSeen.Last), when
|
||||
// the ask is on its stream.
|
||||
func (h heldOnTheBus) seen(ctx context.Context, a delivery.StepAsk, later bool) (build.AskSeen, error) {
|
||||
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
|
||||
module, err := delivery.BuildModuleOf(a.Language)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
return false, err
|
||||
}
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return seen, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
pattern, err := delivery.AskStepWatch(a.ID)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
keys, err := steps.ListKeysFiltered(ctx, pattern)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
return false, err
|
||||
}
|
||||
for range keys.Keys() {
|
||||
keys.Stop()
|
||||
seen.Step = true // a node took it
|
||||
return seen, nil
|
||||
}
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
seen.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return seen, nil
|
||||
return true, nil // a node took it
|
||||
}
|
||||
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
var ahead []build.Queued
|
||||
err = onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seen.OnStream, seen.PutAt = true, q.PutAt
|
||||
return false
|
||||
}
|
||||
ahead = append(ahead, q)
|
||||
return true
|
||||
})
|
||||
if err != nil || !seen.OnStream || !later {
|
||||
return seen, err
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
}
|
||||
seen.Ahead = ahead
|
||||
since := seen.At
|
||||
if since.IsZero() {
|
||||
since = seen.PutAt
|
||||
}
|
||||
for _, q := range ahead {
|
||||
if q.At.Before(since) {
|
||||
since = q.At
|
||||
}
|
||||
}
|
||||
seen.Last = lastTaken(ctx, steps, taken, a.Kind, since)
|
||||
return seen, nil
|
||||
}
|
||||
|
||||
// stream is the build seat's stream of language, its name and the subject of kind on it.
|
||||
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
|
||||
subject, err := delivery.AskSubject(language, kind)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
return false, err
|
||||
}
|
||||
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
|
||||
// messages by sequence and nothing wider (issue 1910).
|
||||
name, err := delivery.AskStream(language)
|
||||
name, err := delivery.AskStream(a.Language)
|
||||
if err != nil {
|
||||
return nil, "", "", err
|
||||
return false, err
|
||||
}
|
||||
stream, err := h.js.Stream(ctx, name)
|
||||
return stream, name, subject, err
|
||||
}
|
||||
|
||||
// onSubject hands each message of subject on stream to each, in order, until each answers false or the subject's
|
||||
// messages end. A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read
|
||||
// by sequence instead, each the next of the subject (one read per message, none for the sequences taken between), and
|
||||
// none consumed.
|
||||
func onSubject(ctx context.Context, stream jetstream.Stream, subject string, each func(build.Queued) bool) error {
|
||||
for seq, n := uint64(1), 0; ; n++ {
|
||||
if n >= maxOnSubject {
|
||||
return fmt.Errorf("%s holds more than %d messages before the one asked after: not read further", subject, maxOnSubject)
|
||||
}
|
||||
m, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(subject))
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q := build.Queued{Seq: m.Sequence, Ask: m.Header.Get(jetstream.MsgIDHeader), PutAt: m.Time}
|
||||
if _, kind, err := delivery.ParseAskSubject(m.Subject); err == nil {
|
||||
q.Kind = kind
|
||||
}
|
||||
if q.At = build.AskedAt(q.Ask); q.At.IsZero() {
|
||||
q.At = q.PutAt
|
||||
}
|
||||
if !each(q) {
|
||||
return nil
|
||||
}
|
||||
seq = m.Sequence + 1
|
||||
}
|
||||
}
|
||||
|
||||
// stepKeys are the build module's step keys by ask id: `<node>.<ask id>` (a node's own key, such as its pause, has no
|
||||
// ask id and is left out).
|
||||
func stepKeys(ctx context.Context, steps jetstream.KeyValue) (map[string][]string, error) {
|
||||
all, err := steps.ListKeys(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return map[string][]string{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer all.Stop()
|
||||
out := map[string][]string{}
|
||||
for k := range all.Keys() {
|
||||
if _, id, ok := strings.Cut(k, "."); ok && !build.AskedAt(id).IsZero() {
|
||||
out[id] = append(out[id], k)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// stepState is the state of the first of keys a node wrote, "" for none.
|
||||
func stepState(ctx context.Context, steps jetstream.KeyValue, keys []string) string {
|
||||
for _, k := range keys {
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if s, err := delivery.DecodeStep(e.Value()); err == nil {
|
||||
return s.State
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// lastTaken is the ask asked last after since that a build node took, of a kind whose taking proves kind passed
|
||||
// (build.Proves), read newest first and at most maxSteps of them; nil for none. A step's tool is "<kind> ask"
|
||||
// (mesh-sdk buildstep).
|
||||
func lastTaken(ctx context.Context, steps jetstream.KeyValue, taken map[string][]string, kind string, since time.Time) *build.Taken {
|
||||
var ids []string
|
||||
for id := range taken {
|
||||
if build.AskedAt(id).After(since) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return build.AskedAt(ids[i]).After(build.AskedAt(ids[j])) })
|
||||
read := 0
|
||||
for _, id := range ids {
|
||||
for _, k := range taken[id] {
|
||||
if read >= maxSteps {
|
||||
return nil
|
||||
}
|
||||
read++
|
||||
e, err := steps.Get(ctx, k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stepKind, ok := strings.CutSuffix(s.Tool, " ask")
|
||||
if !ok || !build.Proves(stepKind, kind) {
|
||||
continue
|
||||
}
|
||||
return &build.Taken{Ask: id, Kind: stepKind, At: build.AskedAt(id), Node: s.Node, Started: s.Started}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drop is build.Dropper's Drop: it removes the ask's message from its build seat's stream, found by its Nats-Msg-Id on its
|
||||
// subject, when the stream still holds it. The builder's account must be granted $JS.API.STREAM.MSG.DELETE.<stream>.
|
||||
func (h heldOnTheBus) Drop(ctx context.Context, a delivery.StepAsk) error {
|
||||
stream, name, subject, err := h.stream(ctx, a.Language, a.Kind)
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return nil
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var seq uint64
|
||||
if err := onSubject(ctx, stream, subject, func(q build.Queued) bool {
|
||||
if q.Ask == a.ID {
|
||||
seq = q.Seq
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); err != nil || seq == 0 {
|
||||
return err
|
||||
}
|
||||
if err := stream.DeleteMsg(ctx, seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
return fmt.Errorf("deleting message %d of %s (is $JS.API.STREAM.MSG.DELETE.%s granted to the builder?): %w", seq, name, name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Queue is what the tool builder_seat_queue answers (build.SeatQueue): the messages of language's build seat's stream, of
|
||||
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
|
||||
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
|
||||
q := build.SeatQueue{Language: language, Note: build.QueueNote}
|
||||
module, err := delivery.BuildModuleOf(language)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
stream, name, _, err := h.stream(ctx, language, delivery.AskDelivery)
|
||||
q.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
q.Note = name + " does not exist"
|
||||
return q, nil
|
||||
}
|
||||
if err != nil {
|
||||
return q, err
|
||||
return false, err
|
||||
}
|
||||
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
|
||||
// instead, each a direct read that consumes nothing.
|
||||
info, err := stream.Info(ctx)
|
||||
if err != nil {
|
||||
return q, err
|
||||
return false, err
|
||||
}
|
||||
q.Messages, q.FirstSeq, q.LastSeq = info.State.Msgs, info.State.FirstSeq, info.State.LastSeq
|
||||
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return q, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
taken, err := stepKeys(ctx, steps)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
for _, kind := range delivery.AskKinds {
|
||||
subject, _ := delivery.AskSubject(language, kind)
|
||||
var ahead []build.Queued
|
||||
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
|
||||
ahead = append(ahead, m)
|
||||
return len(ahead) < limit
|
||||
}); err != nil {
|
||||
return q, err
|
||||
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
|
||||
m, err := stream.GetMsg(ctx, seq)
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
continue // taken and acknowledged, or removed
|
||||
}
|
||||
if len(ahead) == 0 {
|
||||
continue
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for i := range ahead {
|
||||
ahead[i].Step = stepState(ctx, steps, taken[ahead[i].Ask])
|
||||
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
|
||||
return true, nil
|
||||
}
|
||||
seen := build.AskSeen{Kind: kind, Ahead: ahead, Last: lastTaken(ctx, steps, taken, kind, ahead[0].At)}
|
||||
q.Kinds = append(q.Kinds, build.NewKindQueue(kind, seen, time.Now()))
|
||||
}
|
||||
return q, nil
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -24,8 +24,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -164,29 +162,11 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
|
||||
const trunkHead = "3333333333333333333333333333333333333333"
|
||||
|
||||
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
|
||||
func (b *bed) trunk(t *testing.T, head string) uint64 {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rev
|
||||
}
|
||||
|
||||
type bed struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
modules jetstream.KeyValue
|
||||
candidates jetstream.KeyValue
|
||||
source jetstream.KeyValue
|
||||
builds jetstream.KeyValue
|
||||
tools map[string]jetstream.KeyValue
|
||||
instances jetstream.KeyValue
|
||||
@@ -263,7 +243,6 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
}
|
||||
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
|
||||
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
||||
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
|
||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||
b.instances = mk(instancesBucket, 1)
|
||||
@@ -534,9 +513,7 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
|
||||
|
||||
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
|
||||
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
|
||||
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
|
||||
// fingerprint first appeared at).
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
|
||||
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
|
||||
b := startBed(t)
|
||||
other := "src2:" + strings.Repeat("e", 64)
|
||||
@@ -553,7 +530,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
|
||||
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -573,12 +550,12 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
||||
defer b.fake.mu.Unlock()
|
||||
found := false
|
||||
for _, f := range b.fake.fetches {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,7 +603,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
|
||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
|
||||
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
|
||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -824,34 +801,6 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
||||
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
|
||||
}
|
||||
}
|
||||
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
|
||||
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
|
||||
held := heldOnTheBus{b.js}
|
||||
goAsk := delivery.StepAsk{ID: v.Asks[delivery.AskGo], Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
stale := delivery.StepAsk{ID: "mvo53jcm70b5f7", Language: delivery.AskGo, Kind: delivery.AskDelivery}
|
||||
subject, _ := delivery.AskSubject(delivery.AskGo, delivery.AskDelivery)
|
||||
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queued := func() []build.QueuedState {
|
||||
q, err := held.Queue(b.ctx, delivery.AskGo, 10)
|
||||
if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != delivery.AskDelivery {
|
||||
t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err)
|
||||
}
|
||||
return q.Kinds[0].Messages
|
||||
}
|
||||
if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID {
|
||||
t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m)
|
||||
}
|
||||
if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil {
|
||||
t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err)
|
||||
}
|
||||
if err := held.Drop(b.ctx, stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID {
|
||||
t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m)
|
||||
}
|
||||
b.stop()
|
||||
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
|
||||
b.fake.open()
|
||||
|
||||
@@ -13,19 +13,15 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -125,20 +121,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
builds, err := bucket(delivery.BuilderBuildsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build")
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds")
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates, err := bucket(delivery.CandidatesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's.
|
||||
candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf)
|
||||
bound := defaultStepBound
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||
if bound, err = time.ParseDuration(v); err != nil {
|
||||
@@ -161,36 +166,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer calls.Close()
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
CheckAfter: bound,
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||
Held: heldOnTheBus{js},
|
||||
Toolchains: tools,
|
||||
Manifests: manifestReader{manifests},
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
}
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
|
||||
errs := make(chan error, 6)
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
b.Said = func(e build.AskedAgain) {
|
||||
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
}
|
||||
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
||||
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
||||
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
||||
// is asked since R4g, ADR 0524 §7).
|
||||
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
|
||||
case kv == nil:
|
||||
switch kv, err := js.KeyValue(ctx, instancesBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
|
||||
default:
|
||||
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
||||
b.Answered = says.Answered
|
||||
@@ -270,36 +262,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
b.ForgetCandidate(e.Key())
|
||||
}
|
||||
}
|
||||
if candidates != nil {
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
}
|
||||
}
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
|
||||
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
|
||||
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
|
||||
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, rw, eachReAsk) }()
|
||||
}
|
||||
}
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, qw, eachRead) }()
|
||||
}
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
errs := make(chan error, 2)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
b.Wait()
|
||||
@@ -312,50 +282,3 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
|
||||
// delivery.SourceHistory of them: what a candidate's base revision names.
|
||||
type sourceHeads struct {
|
||||
mu sync.Mutex
|
||||
byRev map[uint64]map[string]string
|
||||
}
|
||||
|
||||
// put takes one revision in and answers whether it is new.
|
||||
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
|
||||
if !opOf(e) {
|
||||
return false
|
||||
}
|
||||
src, err := delivery.DecodeSource(e.Value())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.byRev == nil {
|
||||
s.byRev = map[uint64]map[string]string{}
|
||||
}
|
||||
if _, had := s.byRev[e.Revision()]; had {
|
||||
return false
|
||||
}
|
||||
s.byRev[e.Revision()] = src.Heads
|
||||
for len(s.byRev) > delivery.SourceHistory {
|
||||
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
|
||||
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
heads, ok := s.byRev[rev]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
|
||||
}
|
||||
for r, head := range heads {
|
||||
if delivery.SameRepository(r, repository) {
|
||||
return head, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
|
||||
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
|
||||
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
|
||||
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
|
||||
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
|
||||
// and opened again at the next event that needs it where there is one.
|
||||
|
||||
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
|
||||
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
|
||||
// and a candidate's preview waits, saying why.
|
||||
type trunkSource struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
heads *sourceHeads
|
||||
moved func(context.Context)
|
||||
errs chan<- error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
}
|
||||
|
||||
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
|
||||
func (t *trunkSource) ensure() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.opened {
|
||||
return nil
|
||||
}
|
||||
kv, err := t.js.KeyValue(t.ctx, delivery.SourceBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be read (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
w, err := kv.Watch(t.ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched (%v)", delivery.SourceBucket, err)
|
||||
}
|
||||
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
}
|
||||
t.opened = true
|
||||
go func() {
|
||||
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if t.heads.put(e) && t.moved != nil {
|
||||
t.moved(t.ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// start opens the source at start; one that cannot be is said, and the builder goes on.
|
||||
func (t *trunkSource) start() {
|
||||
if err := t.ensure(); err != nil {
|
||||
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
|
||||
}
|
||||
}
|
||||
|
||||
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
|
||||
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if err := t.ensure(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return t.heads.head(ctx, rev, repository)
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// queueReader is the part of the builder a queues value reads through: heldOnTheBus.
|
||||
type queueReader interface {
|
||||
Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error)
|
||||
}
|
||||
|
||||
// queueRead answers one value of the queues state (the tool builder_seat_queue, issue 1945 note 62): a value not yet
|
||||
// answered is read now, from the stream the builder's account may read, and written back by compare-and-set at its
|
||||
// revision with what was read or why it could not be. A value answered already (this builder's own write, or another
|
||||
// instance's) or deleted is left.
|
||||
func queueRead(ctx context.Context, r queueReader, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
a, err := build.DecodeQueueAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not a queue read, left: %v", build.QueuesState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
q, err := r.Queue(read, e.Key(), max(1, min(a.Limit, build.MaxQueueLimit)))
|
||||
cancel()
|
||||
if err != nil {
|
||||
a.Error = err.Error()
|
||||
}
|
||||
a.Queue, a.Answered = &q, time.Now().UTC()
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the read could not be written: %v", build.QueuesState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// askAgainer is the part of the builder a reasks value asks through.
|
||||
type askAgainer interface {
|
||||
AskAgain(key, why string) []string
|
||||
}
|
||||
|
||||
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
|
||||
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
|
||||
// answered already (this builder's own write) or deleted is left.
|
||||
func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
return
|
||||
}
|
||||
r, err := build.DecodeReAsk(e.Value())
|
||||
if err != nil {
|
||||
logf("%s.%s holds what is not an ask to ask again, left: %v", build.ReAsksState, e.Key(), err)
|
||||
return
|
||||
}
|
||||
if r.Answer != "" {
|
||||
return
|
||||
}
|
||||
asked := b.AskAgain(e.Key(), r.Why)
|
||||
if len(asked) == 0 {
|
||||
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
|
||||
} else {
|
||||
r.Answer = "asked again: the " + strings.Join(asked, ", ") + " ask, under a new id the key names"
|
||||
}
|
||||
r.Answered = time.Now().UTC()
|
||||
logf("%s: asked again by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
|
||||
}
|
||||
}
|
||||
|
||||
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
|
||||
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
|
||||
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
|
||||
id := "asked-again-" + e.New
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
return fmt.Errorf("saying %s for %s: %w", build.AskedAgainEvent, e.Key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -28,11 +28,8 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -41,14 +38,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
open := !deleted && openAssignment(value)
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
if g.standing[key] == open {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
g.standing[key] = true
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
return false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,19 +3,10 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -53,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -72,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -97,132 +88,25 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,13 +4,11 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -64,14 +62,11 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -1,337 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
|
||||
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
|
||||
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
|
||||
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
|
||||
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
|
||||
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
|
||||
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
|
||||
|
||||
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
|
||||
const DefaultAskings = 3
|
||||
|
||||
// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
|
||||
const AskedAgainEvent = "build.asked-again"
|
||||
|
||||
// AskedAgain is the body of AskedAgainEvent.
|
||||
type AskedAgain struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
BuildModule string `json:"build_module"`
|
||||
Language string `json:"language"`
|
||||
Old string `json:"old"`
|
||||
New string `json:"new"`
|
||||
Asking int `json:"asking"`
|
||||
Askings int `json:"askings"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
|
||||
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
|
||||
// delivery counts or ack floors (granted to the delivery view alone: $JS.API.CONSUMER.INFO.<stream>.<worker>), so a
|
||||
// message given up on is told by what the workers did after it: a build node took an ask asked later, of the same kind or
|
||||
// of one pulled after it (issue 1945 note 62: seven messages SEAT_NODE_BUILD_GO had given up on sat ahead of a Go ask for
|
||||
// five hours, each counted as waiting, so the ask was never asked again).
|
||||
type AskSeen struct {
|
||||
Stream string
|
||||
// Kind is the ask's kind, At when it was asked (its id's time).
|
||||
Kind string
|
||||
At time.Time
|
||||
// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
|
||||
Step bool
|
||||
// OnStream is whether the stream still holds the ask's message; PutAt is when it was put.
|
||||
OnStream bool
|
||||
PutAt time.Time
|
||||
// Ahead are the messages of its subject before it on the stream.
|
||||
Ahead []Queued
|
||||
// Last is the ask asked last that a build node took (wrote a step for) of the ask's kind or of a kind pulled after
|
||||
// it, when one was taken: every message of the kind asked before it was then delivered already, or given up on.
|
||||
Last *Taken
|
||||
}
|
||||
|
||||
// Queued is one message of a build seat's stream: its sequence, its ask's id and kind, when the ask was asked (its id's
|
||||
// time, the put time when the id says none) and put, and the state of the step a build node wrote for it ("" for none).
|
||||
type Queued struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
PutAt time.Time `json:"put"`
|
||||
Step string `json:"step,omitempty"`
|
||||
}
|
||||
|
||||
// Taken is an ask a build node took: its id, kind and when it was asked, and the step's node and start.
|
||||
type Taken struct {
|
||||
Ask string `json:"ask"`
|
||||
Kind string `json:"kind"`
|
||||
At time.Time `json:"asked"`
|
||||
Node string `json:"node"`
|
||||
Started time.Time `json:"started"`
|
||||
}
|
||||
|
||||
// Proves is whether a taken ask of kind taken shows that a message of kind waiting asked before it is no longer
|
||||
// delivered: a holder pulls the kinds in delivery.AskKinds' order, so taking one of the same kind or of a later one means
|
||||
// its worker had none of waiting's kind left to give.
|
||||
func Proves(taken, waiting string) bool {
|
||||
t, w := slices.Index(delivery.AskKinds, taken), slices.Index(delivery.AskKinds, waiting)
|
||||
return t >= 0 && w >= 0 && t >= w
|
||||
}
|
||||
|
||||
// Waits is whether a message ahead still waits to be delivered: no build node wrote a step for it, and no ask of its
|
||||
// kind asked after it was taken. A message its worker gave up on (delivered its most times) stays on a work queue until
|
||||
// its age, and is not counted.
|
||||
func (s AskSeen) Waits(q Queued) bool {
|
||||
return q.Step == "" && !s.passed(q.At)
|
||||
}
|
||||
|
||||
func (s AskSeen) passed(at time.Time) bool {
|
||||
return s.Last != nil && Proves(s.Last.Kind, s.Kind) && s.Last.At.After(at)
|
||||
}
|
||||
|
||||
// Waiting is how many messages ahead still wait to be delivered (Waits).
|
||||
func (s AskSeen) Waiting() int {
|
||||
n := 0
|
||||
for _, q := range s.Ahead {
|
||||
if s.Waits(q) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind still
|
||||
// deliverable, or no build node took a later ask of its kind (the nodes are busy or paused: a backlog, never a fault,
|
||||
// ADR 0525 §9).
|
||||
func (s AskSeen) Why() string {
|
||||
switch {
|
||||
case s.Step:
|
||||
return ""
|
||||
case !s.OnStream:
|
||||
return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
|
||||
case s.Waiting() > 0 || !s.passed(s.at()):
|
||||
return ""
|
||||
default:
|
||||
behind := ""
|
||||
if n := len(s.Ahead); n > 0 {
|
||||
behind = fmt.Sprintf(" behind %d given up on or taken", n)
|
||||
}
|
||||
return fmt.Sprintf("it waits on %s since %s%s while %s took the %s ask %s asked later at %s: its stream no longer delivers it (given up after its most deliveries)",
|
||||
s.Stream, s.PutAt.UTC().Format(time.RFC3339), behind, s.Last.Node, s.Last.Kind, s.Last.Ask, s.Last.At.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
|
||||
func (s AskSeen) at() time.Time {
|
||||
if s.At.IsZero() {
|
||||
return s.PutAt
|
||||
}
|
||||
return s.At
|
||||
}
|
||||
|
||||
// AskedAt is when an ask id was made (buildstep.NewID: its milliseconds in base 36 before six hex digits), or zero.
|
||||
func AskedAt(id string) time.Time {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.UnixMilli(ms)
|
||||
}
|
||||
|
||||
// Dropper removes an ask's message from its build seat's stream (Builder.Gone may be one): the old message of an ask asked
|
||||
// again, so a message given up on does not sit ahead of every later ask of its kind until its age (issue 1945 note 62).
|
||||
type Dropper interface {
|
||||
Drop(ctx context.Context, a delivery.StepAsk) error
|
||||
}
|
||||
|
||||
// drop removes the old ask's message when Gone can; a refusal is said and changes nothing else: the new ask is put.
|
||||
func (b *Builder) drop(ctx context.Context, ask delivery.StepAsk, old string) {
|
||||
d, ok := b.Gone.(Dropper)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
prior := ask
|
||||
prior.ID = old
|
||||
if err := d.Drop(ctx, prior); err != nil {
|
||||
b.logf("the message of the ask %s asked again as %s stays on its stream: %v", old, ask.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) askings() int {
|
||||
if b.Askings > 0 {
|
||||
return b.Askings
|
||||
}
|
||||
return DefaultAskings
|
||||
}
|
||||
|
||||
var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)
|
||||
|
||||
// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
|
||||
func askingOf(step, buildModule string) int {
|
||||
for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
|
||||
if m[1] == buildModule {
|
||||
if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
|
||||
// languages whose lane waits, none when no lane of this builder waits on the key.
|
||||
func (b *Builder) AskAgain(key, why string) []string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
var asked []string
|
||||
for language, c := range b.waiting[key] {
|
||||
select {
|
||||
case c <- why:
|
||||
default: // one is already asked
|
||||
}
|
||||
asked = append(asked, language)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// Waiting are the build keys whose lanes wait for a step now, each with its languages.
|
||||
func (b *Builder) Waiting() map[string][]string {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
out := map[string][]string{}
|
||||
for key, ls := range b.waiting {
|
||||
for l := range ls {
|
||||
out[key] = append(out[key], l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Builder) listen(key, language string) chan string {
|
||||
b.init()
|
||||
c := make(chan string, 1)
|
||||
b.mu.Lock()
|
||||
if b.waiting[key] == nil {
|
||||
b.waiting[key] = map[string]chan string{}
|
||||
}
|
||||
b.waiting[key][language] = c
|
||||
b.mu.Unlock()
|
||||
return c
|
||||
}
|
||||
|
||||
func (b *Builder) unlisten(key, language string, c chan string) {
|
||||
b.mu.Lock()
|
||||
if b.waiting[key][language] == c {
|
||||
delete(b.waiting[key], language)
|
||||
if len(b.waiting[key]) == 0 {
|
||||
delete(b.waiting, key)
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
var (
|
||||
errCheck = errors.New("the check bound passed")
|
||||
errAgain = errors.New("asked again")
|
||||
)
|
||||
|
||||
type byHand struct{ why string }
|
||||
|
||||
func (h byHand) Error() string { return "asked again by hand: " + h.why }
|
||||
|
||||
// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
|
||||
// asked it again (AskAgain); it answers the step and the ask that has it.
|
||||
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
|
||||
key, _ := delivery.BuilderBuildKey(module, fp)
|
||||
again := b.listen(key, ask.Language)
|
||||
defer b.unlisten(key, ask.Language, again)
|
||||
asking := askingOf(asked.step, l.module)
|
||||
for {
|
||||
step, why, err := b.waitOnce(ctx, ask, again)
|
||||
switch {
|
||||
case errors.Is(err, errCheck):
|
||||
continue
|
||||
case errors.Is(err, errAgain):
|
||||
if asking >= b.askings() {
|
||||
return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
|
||||
}
|
||||
asking++
|
||||
old := ask.ID
|
||||
ask.ID, ask.At = buildstep.NewID(), time.Now()
|
||||
if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
|
||||
return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
|
||||
}
|
||||
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||
return step, ask, err
|
||||
}
|
||||
b.drop(ctx, ask, old)
|
||||
b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
|
||||
if b.Said != nil {
|
||||
b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
|
||||
Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
|
||||
}
|
||||
default:
|
||||
return step, ask, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
|
||||
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
|
||||
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
|
||||
hand, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
go func() {
|
||||
select {
|
||||
case why := <-again:
|
||||
cancel(byHand{why})
|
||||
case <-hand.Done():
|
||||
}
|
||||
}()
|
||||
wait := hand
|
||||
if b.Gone != nil && b.CheckAfter > 0 {
|
||||
var stop context.CancelFunc
|
||||
wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
|
||||
defer stop()
|
||||
}
|
||||
step, err := b.Asks.Wait(wait, ask)
|
||||
if err == nil || ctx.Err() != nil {
|
||||
return step, "", err
|
||||
}
|
||||
var h byHand
|
||||
if errors.As(context.Cause(hand), &h) {
|
||||
return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
|
||||
}
|
||||
if errors.Is(context.Cause(wait), errCheck) {
|
||||
why, gerr := b.Gone.Gone(ctx, ask)
|
||||
if gerr != nil {
|
||||
b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
|
||||
return step, "", errCheck
|
||||
}
|
||||
if why == "" {
|
||||
return step, "", errCheck
|
||||
}
|
||||
return step, why, errAgain
|
||||
}
|
||||
return step, "", err
|
||||
}
|
||||
@@ -73,17 +73,9 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
|
||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
||||
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
||||
}
|
||||
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
|
||||
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
|
||||
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
|
||||
base, err := b.trunkHead(ctx, c.Base, p.Repository)
|
||||
if err != nil {
|
||||
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err)
|
||||
continue
|
||||
}
|
||||
b.mu.Lock()
|
||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
|
||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
b.consider(ctx, name, m)
|
||||
@@ -114,18 +106,3 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]del
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
|
||||
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if b.Trunk == nil {
|
||||
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
|
||||
}
|
||||
head, err := b.Trunk(ctx, rev, repository)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
|
||||
}
|
||||
if head == "" {
|
||||
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
@@ -2,13 +2,9 @@ package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -44,74 +40,11 @@ func candidateValue(t *testing.T, group string, number int, head string) []byte
|
||||
}
|
||||
|
||||
const (
|
||||
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
|
||||
// commit its fingerprint first appeared at.
|
||||
trunkHead = "3333333333333333333333333333333333333333"
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
headX = "1111111111111111111111111111111111111111"
|
||||
headY = "2222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
|
||||
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
|
||||
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w struct {
|
||||
Source struct {
|
||||
Rev uint64 `json:"rev"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
} `json:"source"`
|
||||
Candidate json.RawMessage `json:"candidate"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, err = delivery.DecodeCandidate(w.Candidate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return w.Source.Rev, w.Source.Heads, c, w.Candidate
|
||||
}
|
||||
|
||||
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
|
||||
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
|
||||
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
|
||||
{
|
||||
rev, heads, c, raw := indexerCandidate(t)
|
||||
p := c.Pulls[0]
|
||||
cm := p.Fps["avahi"]
|
||||
ir := newRig(t, goBundle)
|
||||
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
|
||||
if at != rev {
|
||||
return "", fmt.Errorf("source revision %d is not heard", at)
|
||||
}
|
||||
return heads[repository], nil
|
||||
}
|
||||
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
|
||||
value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'),
|
||||
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ir.b.Module(context.Background(), "avahi", value)
|
||||
ir.b.Wait()
|
||||
ir.bm.mu.Lock()
|
||||
ir.bm.calls = nil
|
||||
ir.bm.mu.Unlock()
|
||||
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
|
||||
ir.b.Wait()
|
||||
fetches := ir.bm.callsOf("build_go_fetch")
|
||||
if len(fetches) != 1 {
|
||||
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head {
|
||||
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
|
||||
}
|
||||
}
|
||||
r := newRig(t, goBundle)
|
||||
trunkModule(t, r)
|
||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||
@@ -125,8 +58,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
|
||||
t.Fatalf("%d fetches, want one", len(fetches))
|
||||
}
|
||||
f := fetches[0].body.(map[string]any)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
|
||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
|
||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
|
||||
}
|
||||
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
|
||||
t.Errorf("build args = %+v", b)
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The tool builder_seat_queue reads a build seat's stream as the builder reads it** (novox/hq issue 1945 note 62; the
|
||||
// instruction "tools are how a module is spoken to"): the messages waiting on SEAT_NODE_BUILD_<LANGUAGE>, bounded, each
|
||||
// with its ask, its age, the state of the step a build node wrote for it, and whether the builder counts it as still
|
||||
// waiting (AskSeen.Waits) or as given up on. Only the builder's own account may read the stream (issue 1910): the tools
|
||||
// bundle writes the read into the builder's state queues, keyed by the language, which the builder watches and answers
|
||||
// there, as builder_ask_again does through reasks.
|
||||
const (
|
||||
SeatQueueTool = "builder_seat_queue"
|
||||
// QueuesState is the builder's state of queue reads: the bucket mesh-delivery-builder_queues, keyed by the language.
|
||||
QueuesState = "queues"
|
||||
QueuesBucket = delivery.BuilderModule + "_" + QueuesState
|
||||
// DefaultQueueLimit and MaxQueueLimit bound the messages read of each kind.
|
||||
DefaultQueueLimit = 50
|
||||
MaxQueueLimit = 500
|
||||
)
|
||||
|
||||
// QueueNote says what the read cannot show.
|
||||
const QueueNote = "the builder may not read the seat workers' delivery counts or ack floors ($JS.API.CONSUMER.INFO.<stream>.<worker> is " +
|
||||
"granted to the delivery view alone): a message counts as given up on when no node wrote its step and a node took an ask of its kind " +
|
||||
"asked after it; nats.nats_streams shows the workers' pending and waiting pulls"
|
||||
|
||||
// QueueAsk is a value of the queues state: a read asked (limit, who, when) and, once the builder read the stream, its answer.
|
||||
type QueueAsk struct {
|
||||
V int `json:"v"`
|
||||
Limit int `json:"limit"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Queue *SeatQueue `json:"queue,omitempty"`
|
||||
}
|
||||
|
||||
// SeatQueue is what the builder read of one build seat's stream.
|
||||
type SeatQueue struct {
|
||||
Language string `json:"language"`
|
||||
Stream string `json:"stream"`
|
||||
Messages uint64 `json:"messages"`
|
||||
FirstSeq uint64 `json:"first_seq"`
|
||||
LastSeq uint64 `json:"last_seq"`
|
||||
Kinds []KindQueue `json:"kinds"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// KindQueue is one kind's messages on the stream, oldest first: how many the builder counts as waiting, the ask asked
|
||||
// last a build node took of the kind (or of one pulled after it), and each message.
|
||||
type KindQueue struct {
|
||||
Kind string `json:"kind"`
|
||||
Read int `json:"read"`
|
||||
Waiting int `json:"waiting"`
|
||||
GivenUp int `json:"given_up_or_taken"`
|
||||
Last *Taken `json:"last_taken,omitempty"`
|
||||
Messages []QueuedState `json:"messages"`
|
||||
}
|
||||
|
||||
// QueuedState is one message as the tool shows it.
|
||||
type QueuedState struct {
|
||||
Queued
|
||||
AgeSeconds int64 `json:"age_s"`
|
||||
Waits bool `json:"waits"`
|
||||
}
|
||||
|
||||
// NewKindQueue is seen's messages ahead as one kind's queue at now.
|
||||
func NewKindQueue(kind string, seen AskSeen, now time.Time) KindQueue {
|
||||
k := KindQueue{Kind: kind, Read: len(seen.Ahead), Last: seen.Last}
|
||||
for _, q := range seen.Ahead {
|
||||
w := seen.Waits(q)
|
||||
if w {
|
||||
k.Waiting++
|
||||
} else {
|
||||
k.GivenUp++
|
||||
}
|
||||
k.Messages = append(k.Messages, QueuedState{Queued: q, AgeSeconds: int64(now.Sub(q.PutAt) / time.Second), Waits: w})
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// NewQueueAsk is the value builder_seat_queue writes for a language: its key and the value.
|
||||
func NewQueueAsk(language string, limit int, by string, at time.Time) (string, []byte, error) {
|
||||
if _, err := delivery.AskStream(language); err != nil {
|
||||
return "", nil, fmt.Errorf("language: %w", err)
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = DefaultQueueLimit
|
||||
}
|
||||
data, err := json.Marshal(QueueAsk{V: 1, Limit: min(limit, MaxQueueLimit), By: by, At: at.UTC()})
|
||||
return language, data, err
|
||||
}
|
||||
|
||||
// DecodeQueueAsk reads a queues value.
|
||||
func DecodeQueueAsk(data []byte) (QueueAsk, error) {
|
||||
var a QueueAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a queues value of version %d, not 1", a.V)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// LanguageOf is the language a tool's seat or language argument names: go, node or image, or node-build-<language>.
|
||||
func LanguageOf(arg string) string {
|
||||
return strings.TrimPrefix(strings.TrimSpace(arg), "node-build-")
|
||||
}
|
||||
@@ -379,9 +379,8 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
|
||||
t.Fatal("no fetch")
|
||||
}
|
||||
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
|
||||
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
|
||||
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
|
||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
|
||||
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -53,21 +53,6 @@ type Builder struct {
|
||||
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
||||
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
||||
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
|
||||
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
|
||||
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
|
||||
Gone interface {
|
||||
Gone(ctx context.Context, a delivery.StepAsk) (string, error)
|
||||
}
|
||||
CheckAfter time.Duration
|
||||
// Askings is how many times one build module's part of a build is asked, the first included, before its key is failed
|
||||
// with why; DefaultAskings when 0.
|
||||
Askings int
|
||||
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
|
||||
Said func(AskedAgain)
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -87,8 +72,6 @@ type Builder struct {
|
||||
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||
held map[string]bool
|
||||
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
|
||||
waiting map[string]map[string]chan string
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -105,7 +88,7 @@ func (b *Builder) init() {
|
||||
b.sem = make(chan struct{}, limit)
|
||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.published, b.held = map[string]bool{}, map[string]bool{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -286,13 +269,6 @@ func (b *Builder) Reconsider(ctx context.Context) {
|
||||
for _, k := range sortedKeys(known) {
|
||||
b.consider(ctx, k, known[k])
|
||||
}
|
||||
b.TrunkMoved(ctx)
|
||||
}
|
||||
|
||||
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
|
||||
// revision it was previewed onto had no trunk head to be merged onto.
|
||||
func (b *Builder) TrunkMoved(ctx context.Context) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
cands := make([]delivery.Candidate, 0, len(b.cands))
|
||||
for _, c := range b.cands {
|
||||
@@ -687,14 +663,12 @@ type askNotes struct {
|
||||
b *Builder
|
||||
cl *Claim
|
||||
resumed map[string]string
|
||||
// step is the step words of a key taken over at start: an ask asked again says its asking there (askingOf).
|
||||
step string
|
||||
}
|
||||
|
||||
func (b *Builder) notes(cl *Claim) *askNotes {
|
||||
n := &askNotes{b: b, cl: cl}
|
||||
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
|
||||
n.resumed, n.step = maps.Clone(cl.Build.Asks), cl.Build.Step
|
||||
n.resumed = maps.Clone(cl.Build.Asks)
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -792,7 +766,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
||||
return
|
||||
}
|
||||
}
|
||||
step, ask, err := b.answer(ctx, l, ask, asked, module, m.FP2)
|
||||
step, err := b.Asks.Wait(ctx, ask)
|
||||
if err != nil {
|
||||
l.err = fmt.Errorf("%s: %w", l.module, err)
|
||||
return
|
||||
|
||||
@@ -6,10 +6,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -178,7 +176,6 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
||||
r.b = &Builder{
|
||||
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
||||
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
||||
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
|
||||
}
|
||||
var err error
|
||||
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
||||
@@ -618,286 +615,27 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
|
||||
}
|
||||
}
|
||||
|
||||
// goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to
|
||||
// drop from their stream (Dropper).
|
||||
type goneAsks struct {
|
||||
mu sync.Mutex
|
||||
n int
|
||||
why func(a delivery.StepAsk) string
|
||||
dropped []string
|
||||
}
|
||||
|
||||
func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.dropped = append(g.dropped, a.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) {
|
||||
g.mu.Lock()
|
||||
g.n++
|
||||
g.mu.Unlock()
|
||||
return g.why(a), nil
|
||||
}
|
||||
|
||||
func (g *goneAsks) asked() int {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.n
|
||||
}
|
||||
|
||||
// waitForAsks waits until n asks were put.
|
||||
func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk {
|
||||
t.Helper()
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n)
|
||||
}
|
||||
}
|
||||
return r.asks.Asks()
|
||||
}
|
||||
|
||||
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||
// again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up)
|
||||
// is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way.
|
||||
// again; the key is done when a node takes it.
|
||||
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
t.Run("waits in its stream: never asked again", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
waitForAsks(t, r, 1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no ask was put")
|
||||
}
|
||||
if gone.asked() == 0 {
|
||||
t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
askings int
|
||||
// step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build.
|
||||
step string
|
||||
byHand bool
|
||||
always bool // Gone says every ask cannot be answered, not only the first
|
||||
puts int
|
||||
done bool
|
||||
asking int // the asking the event says, 0 when none is said
|
||||
}{
|
||||
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
|
||||
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
|
||||
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
|
||||
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) { // 1945
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil
|
||||
r.b.Askings = c.askings
|
||||
var said []AskedAgain
|
||||
var mu sync.Mutex
|
||||
r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() }
|
||||
first := ""
|
||||
if c.step != "" {
|
||||
first = "mvl0frv18d1960"
|
||||
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: c.step, Asks: map[string]string{delivery.AskGo: first}})
|
||||
r.st.vals[r.key] = data
|
||||
}
|
||||
gone := &goneAsks{}
|
||||
if !c.byHand {
|
||||
gone.why = func(a delivery.StepAsk) string {
|
||||
if c.always || a.ID == first || first == "" {
|
||||
return "it waits first in line while a node took a later ask"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
if c.byHand {
|
||||
waitForAsks(t, r, 1)
|
||||
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no lane of the key waits")
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.done {
|
||||
asks := waitForAsks(t, r, c.puts)
|
||||
if first == "" && !c.byHand {
|
||||
first = asks[0].ID
|
||||
}
|
||||
last := asks[len(asks)-1]
|
||||
if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") {
|
||||
t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID)
|
||||
}
|
||||
r.asks.End(r.answer(last))
|
||||
}
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if n := len(r.asks.Asks()); n != c.puts {
|
||||
t.Errorf("%d asks put, want %d", n, c.puts)
|
||||
}
|
||||
if c.done && v.State != delivery.BuildDone {
|
||||
t.Errorf("the key = %+v, want done from the ask asked again", v)
|
||||
}
|
||||
if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) {
|
||||
t.Errorf("the key = %+v, want failed saying the ask could not be answered", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
|
||||
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
|
||||
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
|
||||
}
|
||||
gone.mu.Unlock()
|
||||
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
|
||||
t.Errorf("said %+v, want once at asking %d", said, c.asking)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
|
||||
put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
taken := func(kind string, at time.Time) *Taken {
|
||||
return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at}
|
||||
}
|
||||
later := taken(delivery.AskDelivery, put.Add(time.Minute))
|
||||
// ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none).
|
||||
ahead := func(n int, step string) []Queued {
|
||||
var qs []Queued
|
||||
for i := range n {
|
||||
at := put.Add(-time.Duration(n-i) * time.Minute)
|
||||
qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step})
|
||||
}
|
||||
return qs
|
||||
}
|
||||
seen := func(qs []Queued, last *Taken) AskSeen {
|
||||
return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
seen AskSeen
|
||||
want string // a part of the why, "" when it can be answered
|
||||
waiting int
|
||||
}{
|
||||
{"a node took it", AskSeen{Stream: "S", Step: true}, "", 0},
|
||||
{"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0},
|
||||
{"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0},
|
||||
{"first in line, nothing taken since", seen(nil, nil), "", 0},
|
||||
{"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
{"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2},
|
||||
{"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0},
|
||||
// Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours.
|
||||
{"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0},
|
||||
{"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0},
|
||||
{"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0},
|
||||
{"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3},
|
||||
} {
|
||||
got := c.seen.Why()
|
||||
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting {
|
||||
t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting)
|
||||
}
|
||||
}
|
||||
if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() {
|
||||
t.Errorf("AskedAt does not read an ask id's time")
|
||||
}
|
||||
})
|
||||
t.Run("the tool builder_stuck", func(t *testing.T) { // 1945
|
||||
at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC)
|
||||
state := func(vals map[string]string) StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) {
|
||||
var ks []string
|
||||
for k := range vals {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
return ks, nil
|
||||
}, Get: func(k string) (json.RawMessage, time.Time, error) {
|
||||
if v, ok := vals[k]; ok {
|
||||
return json.RawMessage(v), at, nil
|
||||
}
|
||||
return nil, time.Time{}, nil
|
||||
}}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey("m", fp1)
|
||||
done, _ := delivery.BuilderBuildKey("d", fp1)
|
||||
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||
Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}})
|
||||
finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"})
|
||||
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
|
||||
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
|
||||
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
|
||||
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
|
||||
Now: func() time.Time { return at.Add(time.Hour) }}
|
||||
out, err := r.Stuck("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stuck := out["stuck"].([]Stuck)
|
||||
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
|
||||
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
|
||||
}
|
||||
a := stuck[0].Asks
|
||||
if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() ||
|
||||
a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") {
|
||||
t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a)
|
||||
}
|
||||
|
||||
// 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its
|
||||
// own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked
|
||||
// only for the rows kept.
|
||||
var gets, written, inFlight, maxFlight atomic.Int64
|
||||
vals := map[string]string{}
|
||||
for i := 0; i < 3000; i++ {
|
||||
k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i))
|
||||
vals[k] = string(finished)
|
||||
if (i/100)%10 == 0 {
|
||||
vals[k] = string(building)
|
||||
}
|
||||
}
|
||||
slow := state(vals)
|
||||
get := slow.Get
|
||||
slow.Get = func(k string) (json.RawMessage, time.Time, error) {
|
||||
gets.Add(1)
|
||||
n := inFlight.Add(1)
|
||||
for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
inFlight.Add(-1)
|
||||
raw, _, err := get(k)
|
||||
return raw, time.Time{}, err
|
||||
}
|
||||
slow.Written = func(string) time.Time { written.Add(1); return at }
|
||||
for _, row := range []struct {
|
||||
module string
|
||||
stuck int
|
||||
gets int64
|
||||
cut bool
|
||||
}{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} {
|
||||
gets.Store(0)
|
||||
written.Store(0)
|
||||
out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, cut := out["cut"]
|
||||
if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets {
|
||||
t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut)
|
||||
}
|
||||
}
|
||||
if maxFlight.Load() < 2 {
|
||||
t.Errorf("the values were read one at a time")
|
||||
}
|
||||
})
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||
}
|
||||
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||
}
|
||||
}
|
||||
|
||||
// heldAsks says an ask is held when its id is among held.
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
|
||||
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
|
||||
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
|
||||
// builder's state reasks, which the builder watches and answers there.
|
||||
const (
|
||||
StuckTool = "builder_stuck"
|
||||
AskAgainTool = "builder_ask_again"
|
||||
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
||||
ReAsksState = "reasks"
|
||||
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
|
||||
)
|
||||
|
||||
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
|
||||
// read it, its answer.
|
||||
type ReAsk struct {
|
||||
V int `json:"v"`
|
||||
Why string `json:"why"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
}
|
||||
|
||||
// DecodeReAsk reads a reasks value.
|
||||
func DecodeReAsk(data []byte) (ReAsk, error) {
|
||||
var r ReAsk
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return r, err
|
||||
}
|
||||
if r.V != 1 {
|
||||
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
|
||||
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why the build is asked again (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, time.Time, error)
|
||||
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
|
||||
// only for a row they keep, and only when Get did not say.
|
||||
Written func(key string) time.Time
|
||||
}
|
||||
|
||||
// StuckAsk is one ask a key waits on.
|
||||
type StuckAsk struct {
|
||||
Language string `json:"language"`
|
||||
BuildModule string `json:"build_module"`
|
||||
ID string `json:"id"`
|
||||
Asking int `json:"asking"`
|
||||
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
|
||||
// state on its node since its start.
|
||||
Asked time.Time `json:"asked,omitzero"`
|
||||
Step string `json:"step"`
|
||||
}
|
||||
|
||||
// Stuck is one build key that waits on an ask.
|
||||
type Stuck struct {
|
||||
Key string `json:"key"`
|
||||
Module string `json:"module"`
|
||||
FP string `json:"fp"`
|
||||
State string `json:"state"`
|
||||
Step string `json:"step"`
|
||||
By string `json:"by"`
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
Age string `json:"age,omitempty"`
|
||||
Asks []StuckAsk `json:"asks"`
|
||||
ReAsk *ReAsk `json:"reask,omitempty"`
|
||||
Unasked bool `json:"unasked,omitempty"`
|
||||
NoneStep int `json:"asks_with_no_step"`
|
||||
}
|
||||
|
||||
// StuckReader reads what builder_stuck answers.
|
||||
type StuckReader struct {
|
||||
Builds StateRead
|
||||
ReAsks StateRead
|
||||
// Steps are the build modules' steps states, by build module.
|
||||
Steps map[string]StateRead
|
||||
Now func() time.Time
|
||||
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
|
||||
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
|
||||
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
|
||||
const (
|
||||
ReadWorkers = 16
|
||||
MaxReads = 1000
|
||||
)
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
||||
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
||||
started := time.Now()
|
||||
reads := 0
|
||||
defer func() {
|
||||
if r.Logf != nil {
|
||||
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
}()
|
||||
keys, err := r.Builds.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now
|
||||
if r.Now != nil {
|
||||
now = r.Now
|
||||
}
|
||||
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
|
||||
stepKeys := map[string]map[string]string{}
|
||||
for bm, s := range r.Steps {
|
||||
ks, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
|
||||
}
|
||||
byID := make(map[string]string, len(ks))
|
||||
for _, k := range ks {
|
||||
if _, id, err := delivery.ParseStepKey(k); err == nil {
|
||||
byID[id] = k
|
||||
}
|
||||
}
|
||||
stepKeys[bm] = byID
|
||||
}
|
||||
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
|
||||
// parallel, at most MaxReads of them.
|
||||
type parsed struct{ key, mod, fp string }
|
||||
var matching []parsed
|
||||
for _, key := range keys {
|
||||
mod, fp, err := delivery.ParseBuilderBuildKey(key)
|
||||
if err != nil || (module != "" && mod != module) {
|
||||
continue
|
||||
}
|
||||
matching = append(matching, parsed{key, mod, fp})
|
||||
}
|
||||
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
|
||||
cut := len(matching) > MaxReads
|
||||
total := len(matching)
|
||||
if cut {
|
||||
matching = matching[:MaxReads]
|
||||
}
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
ats := make([]time.Time, len(matching))
|
||||
if err := parallel(len(matching), func(i int) error {
|
||||
raw, at, err := r.Builds.Get(matching[i].key)
|
||||
if err == nil {
|
||||
values[i], ats[i] = raw, at
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reads += len(matching)
|
||||
out := []Stuck{}
|
||||
for i, m := range matching {
|
||||
key, mod, fp := m.key, m.mod, m.fp
|
||||
raw, at := values[i], ats[i]
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
v, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key)
|
||||
}
|
||||
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
|
||||
if !at.IsZero() {
|
||||
s.Age = now().Sub(at).Round(time.Second).String()
|
||||
}
|
||||
for language, id := range v.Asks {
|
||||
bm, _ := delivery.BuildModuleOf(language)
|
||||
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
|
||||
if t, ok := idTime(id); ok {
|
||||
a.Asked = t
|
||||
}
|
||||
if st, ok := r.Steps[bm]; ok {
|
||||
if k, ok := stepKeys[bm][id]; ok {
|
||||
reads++
|
||||
if raw, _, err := st.Get(k); err == nil && raw != nil {
|
||||
if step, err := delivery.DecodeStep(raw); err == nil {
|
||||
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if a.Step == "none" {
|
||||
s.NoneStep++
|
||||
}
|
||||
s.Asks = append(s.Asks, a)
|
||||
}
|
||||
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
|
||||
s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
|
||||
if r.ReAsks.Get != nil {
|
||||
reads++
|
||||
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
|
||||
if ra, err := DecodeReAsk(raw); err == nil {
|
||||
s.ReAsk = &ra
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if !out[i].Since.Equal(out[j].Since) {
|
||||
return out[i].Since.Before(out[j].Since)
|
||||
}
|
||||
return out[i].Key < out[j].Key
|
||||
})
|
||||
answer := map[string]any{"stuck": out, "count": len(out),
|
||||
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
||||
if cut {
|
||||
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// StuckBuildModules are the build modules whose steps builder_stuck reads.
|
||||
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
|
||||
|
||||
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
|
||||
func idTime(id string) (time.Time, bool) {
|
||||
if len(id) <= 6 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
||||
if err != nil || ms <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), true
|
||||
}
|
||||
@@ -23,21 +23,18 @@ type state struct {
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Tools []string `json:"tools"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
@@ -87,13 +84,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||
@@ -115,14 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
t.Errorf("emits = %v, want %v", m.Emits, want)
|
||||
}
|
||||
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||
}
|
||||
@@ -137,7 +127,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the builder's tools are its bundle's, and it holds no seat", r)
|
||||
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
@@ -159,13 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
} else if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
||||
t.Errorf("the tools' source %s: %v", a.From, err)
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
||||
|
||||
@@ -2,14 +2,6 @@
|
||||
"module": "mesh-delivery-builder",
|
||||
"version": "1",
|
||||
"slug": "builder",
|
||||
"tools": [
|
||||
"builder_ask_again",
|
||||
"builder_seat_queue",
|
||||
"builder_stuck"
|
||||
],
|
||||
"emits": [
|
||||
"build.asked-again"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
@@ -20,9 +12,7 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 }
|
||||
{ "name": "instances", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
@@ -31,7 +21,6 @@
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.assignments",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules"
|
||||
@@ -88,17 +77,6 @@
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder",
|
||||
"binary": "mesh-delivery-builder"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder-tools",
|
||||
"binary": "mesh-delivery-builder-tools",
|
||||
"loads": [
|
||||
"mesh-delivery-builder-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -16,35 +15,8 @@ import (
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// reads counts the values this call read from the bus: the runtime launches the tools for one call, so the count is the call's.
|
||||
var reads atomic.Int64
|
||||
|
||||
// cutAnswer says in the answer that it was cut, and how to narrow it.
|
||||
func cutAnswer(answer map[string]any, cut string) map[string]any {
|
||||
if cut != "" {
|
||||
answer["cut"], answer["note"] = true, cut
|
||||
}
|
||||
return answer
|
||||
}
|
||||
|
||||
// logged runs every tool with one line to stderr per call: the tool, the values it read and the time it took.
|
||||
func logged(tools []stdio.Tool) []stdio.Tool {
|
||||
for i := range tools {
|
||||
name, run := tools[i].Name, tools[i].Run
|
||||
tools[i].Run = func(args map[string]any) (any, error) {
|
||||
reads.Store(0)
|
||||
start := time.Now()
|
||||
out, err := run(args)
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %s read %d values in %s (error: %v)\n", name, reads.Load(), time.Since(start).Round(time.Millisecond), err)
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
return tools
|
||||
}
|
||||
|
||||
func read(s stdio.KeptState) check.StateRead {
|
||||
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||
reads.Add(1)
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
@@ -128,8 +100,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
group, _ := args["group"].(string)
|
||||
vs, cut, err := r.Verdicts(group)
|
||||
return cutAnswer(map[string]any{"verdicts": vs}, cut), err
|
||||
vs, err := r.Verdicts(group)
|
||||
return map[string]any{"verdicts": vs}, err
|
||||
}},
|
||||
{Name: check.PullReadTool,
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||
@@ -150,8 +122,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
level, _ := args["level"].(string)
|
||||
digest, _ := args["digest"].(string)
|
||||
kept, cut, err := r.Kept(level, digest)
|
||||
return cutAnswer(map[string]any{"kept": kept}, cut), err
|
||||
kept, err := r.Kept(level, digest)
|
||||
return map[string]any{"kept": kept}, err
|
||||
}},
|
||||
{Name: check.PreviewsReadTool,
|
||||
Description: "The merge previews the checker holds, each as stored under previews.<repository>.<head>.<trunk>: the merge commit, per module directory " +
|
||||
@@ -166,8 +138,8 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
if head != "" && len(head) < 7 {
|
||||
return nil, fmt.Errorf("the head %q is too short: give at least 7 characters", head)
|
||||
}
|
||||
ps, bad, cut, err := r.Previews(repository, head)
|
||||
return cutAnswer(map[string]any{"previews": ps, "unreadable": bad}, cut), err
|
||||
ps, bad, err := r.Previews(repository, head)
|
||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||
}},
|
||||
{Name: check.StatusReadTool,
|
||||
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
||||
@@ -212,7 +184,7 @@ func main() {
|
||||
}},
|
||||
Trunks: trunks,
|
||||
}
|
||||
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
|
||||
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -182,8 +182,7 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
|
||||
return b, err == nil, err
|
||||
}
|
||||
|
||||
// CommitOf is the commit the indexer's record of the module names: the commit its fingerprint first appeared at, so the source
|
||||
// of the build the mesh runs (a later merge that leaves the module alone writes no record, the operator 2026-10-25).
|
||||
// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs.
|
||||
func (r *repoModules) CommitOf(module string) string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -152,12 +152,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
|
||||
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
buckets := map[string]jetstream.KeyValue{}
|
||||
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
|
||||
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} {
|
||||
if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil {
|
||||
if buckets[name], err = bucket(name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -207,7 +212,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
||||
// says nothing of the builds, until the checker starts again.
|
||||
var builds check.BuildStates
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
|
||||
switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
|
||||
default:
|
||||
builds = buildReader{kvStore{kv}}
|
||||
}
|
||||
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
|
||||
@@ -280,38 +290,47 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
|
||||
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
|
||||
// issue 1912), so the node-engine's have is not watched here.
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.PresenceBucket, "no node counts as online until the checker starts again", logger.Printf); kv != nil {
|
||||
if pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.PresenceBucket, err, "no node counts as online until the checker starts again")
|
||||
} else {
|
||||
go func() {
|
||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||
}
|
||||
})
|
||||
}()
|
||||
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
|
||||
default:
|
||||
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
|
||||
// yet no trunk is known, so nothing is checked.
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no trunk is known, so nothing is checked until the checker starts again", logger.Printf); kv != nil {
|
||||
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
default:
|
||||
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey {
|
||||
trunks.Put(e.Value(), !opOf(e))
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no trunk is known, so nothing is checked until the checker starts again")
|
||||
} else {
|
||||
go func() {
|
||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
||||
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
||||
}
|
||||
})
|
||||
}()
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
|
||||
ch.TrunkMoved(ctx) // the trunk moved: its statuses go pending, and a verdict for the old trunk is judged again
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
|
||||
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
|
||||
|
||||
@@ -28,11 +28,8 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -41,14 +38,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
open := !deleted && openAssignment(value)
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
if g.standing[key] == open {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
g.standing[key] = true
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
return false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,19 +3,10 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
@@ -53,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -72,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -97,122 +88,25 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
if n := len(set.Entries("typescript")); n != 2 {
|
||||
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||
}
|
||||
|
||||
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
|
||||
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
|
||||
// is an error naming the bucket and its grant.
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
|
||||
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
|
||||
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
|
||||
}
|
||||
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
|
||||
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Errorf("the refusals were not said: %q", logs.String())
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
||||
type refusingBus struct {
|
||||
refuseOpen bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
||||
if r.refuseOpen {
|
||||
return nil, r.err
|
||||
}
|
||||
return refusedWatch{err: r.err}, nil
|
||||
}
|
||||
|
||||
type refusedWatch struct {
|
||||
jetstream.KeyValue
|
||||
err error
|
||||
}
|
||||
|
||||
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
return nil, r.err
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -65,14 +63,11 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d h1:oJREw0ggKqJWJ5DEISSL0i2ZC5XqdXAidxo9NJ74TCc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024214154-bc16740feb0d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// countingAsks puts asks through inner, and answers no wait until want asks are put (or a bound passes, when it says
|
||||
// how many were): a group whose asks are put one after another, each waited for, never gets past its first.
|
||||
type countingAsks struct {
|
||||
inner Asks
|
||||
want int
|
||||
mu sync.Mutex
|
||||
puts int
|
||||
all chan struct{}
|
||||
}
|
||||
|
||||
func (c *countingAsks) Put(ctx context.Context, a delivery.StepAsk) error {
|
||||
if err := c.inner.Put(ctx, a); err != nil {
|
||||
return err
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.puts++
|
||||
if c.puts == c.want {
|
||||
close(c.all)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *countingAsks) Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) {
|
||||
select {
|
||||
case <-c.all:
|
||||
case <-time.After(time.Second):
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return delivery.Step{}, fmt.Errorf("%d of %d asks were put before the first was waited for", c.puts, c.want)
|
||||
}
|
||||
return c.inner.Wait(ctx, a)
|
||||
}
|
||||
|
||||
// 1921/S8: a group's asks are all put before any is waited for, so its members run side by side on the build nodes;
|
||||
// the verdict is judged in member order as before.
|
||||
func TestAGroupPutsEveryMembersAskBeforeTheFirstAnswer(t *testing.T) {
|
||||
b := newBench(t)
|
||||
counting := &countingAsks{inner: b.ex.Asks, want: 3, all: make(chan struct{})}
|
||||
b.ex.Asks = counting
|
||||
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf("novox/mesh-controller", 3, head1))
|
||||
v := b.judge(t, cs).Verdict
|
||||
if v.State != delivery.VerdictPass {
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
if counting.puts != 3 {
|
||||
t.Errorf("%d asks were put, want 3", counting.puts)
|
||||
}
|
||||
}
|
||||
|
||||
// 1921/S10: the levels of an ask run side by side, so what of the ask's time is its fetch (Took.Clone) is the ask's time
|
||||
// less its longest level, not less the sum of its levels.
|
||||
func TestTheCloneTimeTakesTheLongestLevelOffNotTheSum(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.test = func(c stepCall) (any, error) {
|
||||
r := captured[Tested](t, "go-test-slow")
|
||||
r.Level, _ = c.args["level"].(string)
|
||||
r.Elapsed, r.Setup, r.Lint = 1.5, 0, 0 // the ask's step takes 2 s (bench.answer)
|
||||
return r, nil
|
||||
}
|
||||
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||
if v.State != delivery.VerdictPass || v.Took.Clone != 500 {
|
||||
t.Errorf("clone = %d ms, want 500 (2 s less the longest level's 1.5 s): %+v", v.Took.Clone, v)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0542 §2: a lint finding, which the Go build module marks as the lint step's, is reported as the lint step's failure,
|
||||
// not as the unit level's.
|
||||
func TestALintFindingIsTheLintStepsFailure(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.test = func(c stepCall) (any, error) {
|
||||
r := captured[Tested](t, "go-test-slow")
|
||||
r.Level, _ = c.args["level"].(string)
|
||||
if r.Level == delivery.LevelUnit {
|
||||
r.Passed = false
|
||||
r.Failed = []Failure{{Package: ".", Test: "gofmt", Step: "lint", Output: "not as gofmt writes them (run gofmt -w): a.go"}}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||
if v.State != delivery.VerdictFail || v.Failed.Test != "lint: ..gofmt" {
|
||||
t.Errorf("failed = %+v, want the lint step's failure", v.Failed)
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
|
||||
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
|
||||
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
|
||||
// through mesh-sdk (rules-are-weighed).
|
||||
const ReadWorkers = 16
|
||||
|
||||
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
|
||||
const MaxReads = 1000
|
||||
|
||||
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
|
||||
// further read starts.
|
||||
func parallel(n int, read func(i int) error) error {
|
||||
var (
|
||||
next atomic.Int64
|
||||
failed atomic.Bool
|
||||
once sync.Once
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for w := 0; w < min(ReadWorkers, n); w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for !failed.Load() {
|
||||
i := int(next.Add(1) - 1)
|
||||
if i >= n {
|
||||
return
|
||||
}
|
||||
if err := read(i); err != nil {
|
||||
once.Do(func() { first = err })
|
||||
failed.Store(true)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return first
|
||||
}
|
||||
|
||||
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
|
||||
func capKeys(keys []string, narrow string) ([]string, string) {
|
||||
if len(keys) <= MaxReads {
|
||||
return keys, ""
|
||||
}
|
||||
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
|
||||
}
|
||||
|
||||
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
|
||||
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
|
||||
func newestFirst(keys []string) {
|
||||
id := func(k string) (string, bool) {
|
||||
_, id, err := delivery.ParseStepKey(k)
|
||||
return id, err == nil
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
a, aok := id(keys[i])
|
||||
b, bok := id(keys[j])
|
||||
switch {
|
||||
case aok != bok:
|
||||
return aok
|
||||
case len(a) != len(b):
|
||||
return len(a) > len(b)
|
||||
case a != b:
|
||||
return a > b
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
}
|
||||
@@ -82,6 +82,9 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
||||
}
|
||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Merged {
|
||||
continue // merged while the group waits for its other members: judged by the verdict the group had (issue 1791)
|
||||
}
|
||||
switch head, known := trunk[p.Repository]; {
|
||||
case !known || head == "":
|
||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
|
||||
@@ -119,7 +122,18 @@ func GateHeads(c delivery.Candidate) string {
|
||||
}
|
||||
|
||||
// sameTrunk says whether two verdicts' trunk heads are the same.
|
||||
func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) }
|
||||
// sameTrunk says whether a verdict judged on the trunk heads judged stands for a case on the trunk heads now: every
|
||||
// repository of the case at the head it was judged on. A repository the verdict names and the case does not is a member
|
||||
// that merged since (issue 1791): its own merge moved that trunk, and the members left are judged by the verdict the
|
||||
// group had.
|
||||
func sameTrunk(judged, now map[string]string) bool {
|
||||
for repo, head := range now {
|
||||
if judged[repo] != head {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(now) > 0 || len(judged) == 0
|
||||
}
|
||||
|
||||
// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in).
|
||||
type Trunks struct {
|
||||
|
||||
@@ -83,12 +83,24 @@ type Checker struct {
|
||||
tried map[string]bool // by verdict key: looked at since this process started
|
||||
epoch uint64 // changes of the nodes seen since this process started
|
||||
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
|
||||
// current is, by candidates key, the verdict key of the heads the group holds now; running holds, by candidates key,
|
||||
// the stop of each check that runs, by its verdict key (issue 1791; issue 1921, try speed S9).
|
||||
current map[string]string
|
||||
running map[string]map[string]context.CancelCauseFunc
|
||||
}
|
||||
|
||||
// superseded is why a check whose heads the group no longer holds was stopped: the heads it holds now.
|
||||
type superseded struct{ by string }
|
||||
|
||||
func (s superseded) Error() string {
|
||||
return "superseded: the group holds other heads now (" + s.by + "), whose check judges it"
|
||||
}
|
||||
|
||||
func (ch *Checker) init() {
|
||||
ch.once.Do(func() {
|
||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||
ch.nodesFail = map[string]uint64{}
|
||||
ch.current, ch.running = map[string]string{}, map[string]map[string]context.CancelCauseFunc{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -126,6 +138,15 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
|
||||
}
|
||||
ch.mu.Lock()
|
||||
ch.known[key] = c
|
||||
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil {
|
||||
ch.current[key] = vk
|
||||
// A check of heads the group no longer holds is stopped: its asks are not put, and it ends superseded.
|
||||
for running, stop := range ch.running[key] {
|
||||
if running != vk {
|
||||
stop(superseded{by: c.HeadsHash})
|
||||
}
|
||||
}
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
ch.consider(ctx, key, c)
|
||||
}
|
||||
@@ -329,12 +350,48 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
run, stop := context.WithCancelCause(ctx)
|
||||
defer stop(nil)
|
||||
ch.mu.Lock()
|
||||
now := ch.current[key]
|
||||
if ch.running[key] == nil {
|
||||
ch.running[key] = map[string]context.CancelCauseFunc{}
|
||||
}
|
||||
ch.running[key][cs.Key] = stop
|
||||
ch.mu.Unlock()
|
||||
defer func() {
|
||||
ch.mu.Lock()
|
||||
delete(ch.running[key], cs.Key)
|
||||
if len(ch.running[key]) == 0 {
|
||||
delete(ch.running, key)
|
||||
}
|
||||
ch.mu.Unlock()
|
||||
}()
|
||||
if now != "" && now != cs.Key {
|
||||
stop(superseded{by: now}) // the group moved on before the check began
|
||||
}
|
||||
began := ch.now()
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
out := ch.judge(ctx, cs)
|
||||
if run.Err() == nil {
|
||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||
}
|
||||
out := ch.judge(run, cs)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var sup superseded
|
||||
if errors.As(context.Cause(run), &sup) {
|
||||
// Stopped for newer heads: the verdict ends failed, not running, and no status is put, as the newer check owns the
|
||||
// status of the heads they share.
|
||||
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
|
||||
Failed: delivery.FailedFirst{Machine: "checker", Words: sup.Error()}}
|
||||
if data, err := delivery.EncodeVerdict(v); err == nil {
|
||||
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
|
||||
ch.logf("%s: the superseded verdict could not be written: %v", cs.Key, err)
|
||||
}
|
||||
}
|
||||
ch.logf("%s: %s", cs.Key, sup.Error())
|
||||
return
|
||||
}
|
||||
v := out.Verdict
|
||||
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
|
||||
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// failingResult is a level's test result in which the named tests failed, each in its own package.
|
||||
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
|
||||
t.Helper()
|
||||
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
|
||||
return v, st
|
||||
}
|
||||
|
||||
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
|
||||
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
|
||||
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
|
||||
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
|
||||
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
|
||||
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
|
||||
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
|
||||
for name, tests := range map[string][]json.RawMessage{
|
||||
"one result": {failingResult(t, 10, a, b)},
|
||||
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
|
||||
} {
|
||||
v, st := judgeLevel(t, tests...)
|
||||
if v == nil {
|
||||
t.Fatalf("%s: the level passed", name)
|
||||
}
|
||||
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
|
||||
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
|
||||
t.Errorf("%s: verdict %+v", name, v)
|
||||
}
|
||||
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
|
||||
t.Errorf("%s: %d tests counted, want all 10", name, run)
|
||||
}
|
||||
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
|
||||
t.Errorf("%s: the verdict is not written: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
|
||||
// failed, and each entry's line is cut to MaxFailureLine.
|
||||
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
|
||||
var fs []Failure
|
||||
for i := range 60 {
|
||||
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
|
||||
}
|
||||
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
|
||||
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
|
||||
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
|
||||
}
|
||||
if len(v.Failures[0].Words) > MaxFailureLine {
|
||||
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
|
||||
}
|
||||
if n := failedCount(*v); n != 60 {
|
||||
t.Errorf("failedCount = %d, want 60", n)
|
||||
}
|
||||
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
|
||||
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
|
||||
t.Errorf("line %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
|
||||
// before.
|
||||
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
|
||||
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
|
||||
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
|
||||
t.Fatalf("%q", got)
|
||||
}
|
||||
}
|
||||
@@ -74,6 +74,18 @@ func (f *fakeSteps) tool(name string) []stepCall {
|
||||
return out
|
||||
}
|
||||
|
||||
// toolAt is tool's calls of one level: a check's levels run side by side (issue 1921, try speed S10), so only the calls of
|
||||
// one level keep an order.
|
||||
func (f *fakeSteps) toolAt(name, level string) []stepCall {
|
||||
var out []stepCall
|
||||
for _, c := range f.tool(name) {
|
||||
if c.args["level"] == level {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type entries []delivery.NodeToolchain
|
||||
|
||||
func (e entries) Entries(language string) []delivery.NodeToolchain {
|
||||
@@ -269,9 +281,9 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
tests := b.steps.tool("test")
|
||||
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || tests[0].args["level"] != "unit" {
|
||||
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, then the integration level", tests)
|
||||
tests := b.steps.toolAt("test", "unit")
|
||||
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || len(b.steps.toolAt("test", "integration")) != 1 {
|
||||
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, and the integration level", b.steps.tool("test"))
|
||||
}
|
||||
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
|
||||
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
|
||||
@@ -302,7 +314,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
||||
// Asked again with nothing changed: only the failed package runs; the good one is taken.
|
||||
b.steps.calls = nil
|
||||
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
tests := b.steps.tool("test")
|
||||
tests := b.steps.toolAt("test", "unit")
|
||||
if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" {
|
||||
t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests)
|
||||
}
|
||||
@@ -310,7 +322,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
|
||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}}
|
||||
b.steps.calls = nil
|
||||
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
tests = b.steps.tool("test")
|
||||
tests = b.steps.toolAt("test", "unit")
|
||||
if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" {
|
||||
t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got)
|
||||
}
|
||||
@@ -341,7 +353,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
|
||||
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
|
||||
// after it (so one check names every failure); the later level ran beside it (S10) and is not judged.
|
||||
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
|
||||
b := newBench(t)
|
||||
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
|
||||
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
|
||||
@@ -356,12 +370,12 @@ func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { //
|
||||
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
|
||||
t.Fatalf("verdict = %+v", out.Verdict)
|
||||
}
|
||||
tests := b.steps.tool("test")
|
||||
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
|
||||
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
|
||||
tests := b.steps.toolAt("test", "unit")
|
||||
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
|
||||
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
|
||||
}
|
||||
if n := len(b.steps.tool("digest")); n != 1 {
|
||||
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
|
||||
if out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
|
||||
t.Errorf("the integration level was judged after the unit level failed: %+v", out.Verdict.Levels)
|
||||
}
|
||||
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
|
||||
t.Errorf("composition = %+v", out.Verdict.Composition)
|
||||
|
||||
@@ -159,6 +159,9 @@ type target struct {
|
||||
runner string
|
||||
// judged is what the line says of the program that judged a change to the build module; empty for any other change.
|
||||
judged string
|
||||
// ask is the check ask put for it, and failed why it could not be asked (issue 1921, try speed S8).
|
||||
ask delivery.StepAsk
|
||||
failed *delivery.Verdict
|
||||
}
|
||||
|
||||
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
|
||||
@@ -307,12 +310,13 @@ func (st *state) levelsOf() []string {
|
||||
return []string{delivery.LevelUnit, delivery.LevelIntegration}
|
||||
}
|
||||
|
||||
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
|
||||
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
||||
// (ADR 0484 clause 13).
|
||||
// judgeAll puts one check ask per pull request and build module the repository needs, every ask of the group at once so
|
||||
// the members run side by side on the build nodes (issue 1921, try speed S8), and judges the answers in member order, the
|
||||
// mesh-sdk member first. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
||||
// (ADR 0484 clause 13): the verdict is the first failure in member order, as when the asks were put one after another.
|
||||
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
||||
// Node fetches take the group's package (ADR 0500 §5).
|
||||
// Node fetches take the group's package (ADR 0500 §5). Those Node asks alone wait for the mesh-sdk member's answer.
|
||||
pulls := slices.Clone(st.c.Pulls)
|
||||
slices.SortStableFunc(pulls, func(a, b Pull) int {
|
||||
if (a.Repository == SDKRepository) != (b.Repository == SDKRepository) {
|
||||
@@ -323,34 +327,32 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
}
|
||||
return 0
|
||||
})
|
||||
var groupPackage map[string]any
|
||||
for _, p := range pulls {
|
||||
var merged []Fetched
|
||||
sdkGroup := len(st.c.Pulls) > 1 && len(pulls) > 0 && pulls[0].Repository == SDKRepository
|
||||
members := make([][]*target, len(pulls))
|
||||
var waiting []*target // Node asks of the other members, put once the mesh-sdk member is judged
|
||||
for i, p := range pulls {
|
||||
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
||||
t := &target{pull: p, module: bm}
|
||||
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
||||
t.toolchain = tc.Version
|
||||
}
|
||||
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
||||
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
|
||||
args["package"] = groupPackage
|
||||
}
|
||||
if bm == buildrun.GoBuildModule {
|
||||
beside, err := st.j.besideFor(ctx, st.c, p)
|
||||
if err != nil {
|
||||
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
||||
}
|
||||
if len(beside) > 0 {
|
||||
args["beside"] = beside
|
||||
t.beside = besideLines(beside)
|
||||
}
|
||||
if p.Repository == catalogueRepository {
|
||||
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||
}
|
||||
}
|
||||
t.fetchArgs = args
|
||||
t := st.target(ctx, p, bm)
|
||||
members[i] = append(members[i], t)
|
||||
st.targets = append(st.targets, t)
|
||||
if v := st.ask(ctx, t, &merged); v != nil {
|
||||
if t.failed != nil {
|
||||
continue // said when the judging reaches it, so the verdict stays the first failure in member order
|
||||
}
|
||||
if sdkGroup && bm == buildrun.NodeBuildModule && p.Repository != SDKRepository {
|
||||
waiting = append(waiting, t)
|
||||
continue
|
||||
}
|
||||
t.failed = st.put(ctx, t)
|
||||
}
|
||||
}
|
||||
var groupPackage map[string]any
|
||||
for i, p := range pulls {
|
||||
var merged []Fetched
|
||||
for _, t := range members[i] {
|
||||
if t.failed != nil {
|
||||
return t.failed
|
||||
}
|
||||
if v := st.judgeAsk(ctx, t, &merged); v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
@@ -360,14 +362,51 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
|
||||
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
|
||||
}
|
||||
if i == 0 && sdkGroup {
|
||||
for _, t := range waiting {
|
||||
if groupPackage != nil {
|
||||
t.fetchArgs["package"] = groupPackage
|
||||
}
|
||||
t.failed = st.put(ctx, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
||||
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
|
||||
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||
// target is the check of one pull request by one build module: its fetch's arguments, its toolchain, the siblings its
|
||||
// tests read and the runner it is asked with; failed says why it cannot be asked.
|
||||
func (st *state) target(ctx context.Context, p Pull, bm string) *target {
|
||||
t := &target{pull: p, module: bm}
|
||||
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
||||
t.toolchain = tc.Version
|
||||
}
|
||||
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
||||
if bm == buildrun.GoBuildModule {
|
||||
beside, err := st.j.besideFor(ctx, st.c, p)
|
||||
if err != nil {
|
||||
t.failed = fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
||||
return t
|
||||
}
|
||||
if len(beside) > 0 {
|
||||
args["beside"] = beside
|
||||
t.beside = besideLines(beside)
|
||||
}
|
||||
if p.Repository == catalogueRepository {
|
||||
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||
}
|
||||
}
|
||||
t.fetchArgs = args
|
||||
return t
|
||||
}
|
||||
|
||||
// put puts the target's check ask on its build seat's stream; a check that was stopped (its group holds other heads now,
|
||||
// issue 1791) puts none.
|
||||
func (st *state) put(ctx context.Context, t *target) *delivery.Verdict {
|
||||
p := t.pull
|
||||
if err := context.Cause(ctx); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d was not asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
||||
if t.runner != "" {
|
||||
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||
@@ -379,13 +418,20 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
if st.c.Kind == KindTrunk {
|
||||
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
||||
}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||
t.ask = delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Check: job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, t.ask.ID, p.Repository, p.Number), "")
|
||||
if err := st.j.Asks.Put(ctx, t.ask); err != nil {
|
||||
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
return nil
|
||||
}
|
||||
|
||||
// judgeAsk waits for the step of the node that pulled the target's ask and judges its answer: the fetch, the pins of its
|
||||
// go.mod files, then each level's digest, kept verdicts and tests.
|
||||
func (st *state) judgeAsk(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||
p := t.pull
|
||||
step, err := st.j.Asks.Wait(ctx, t.ask)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fail(t.module, ctx.Err().Error())
|
||||
@@ -427,7 +473,7 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
ran := 0.0
|
||||
for _, la := range answer.Levels {
|
||||
v, elapsed := st.level(ctx, t, la)
|
||||
ran += elapsed
|
||||
ran = max(ran, elapsed) // the levels run side by side (issue 1921, try speed S10): the longest is what they took
|
||||
if v != nil {
|
||||
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||
return v
|
||||
@@ -438,8 +484,9 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
|
||||
}
|
||||
|
||||
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
||||
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
|
||||
// verdict. It answers the level's running time (its tests' and setup's).
|
||||
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
|
||||
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
|
||||
// running time (its tests' and setup's).
|
||||
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
||||
level := la.Level
|
||||
a := st.levels[level]
|
||||
@@ -461,6 +508,7 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
||||
a.kept += k.Tests
|
||||
}
|
||||
elapsed := 0.0
|
||||
var failedResults []Tested
|
||||
for _, raw := range la.Tests {
|
||||
var res Tested
|
||||
if err := json.Unmarshal(raw, &res); err != nil {
|
||||
@@ -489,9 +537,12 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
|
||||
}
|
||||
}
|
||||
if !res.Passed {
|
||||
return levelFailure(t, level, res), elapsed
|
||||
failedResults = append(failedResults, res)
|
||||
}
|
||||
}
|
||||
if len(failedResults) > 0 {
|
||||
return levelFailure(t, level, failedResults...), elapsed
|
||||
}
|
||||
return nil, elapsed
|
||||
}
|
||||
|
||||
@@ -531,98 +582,23 @@ const MaxWords = 4096
|
||||
|
||||
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
|
||||
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
|
||||
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
|
||||
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
|
||||
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
|
||||
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
|
||||
// status line, which is cut short, names it.
|
||||
func failureWords(f Failure) string {
|
||||
out, passed := dropSummary(strings.TrimSpace(f.Output))
|
||||
cause := strings.TrimSpace(f.Cause)
|
||||
if cause == "" {
|
||||
cause = causeIn(out)
|
||||
}
|
||||
switch {
|
||||
case cause == "" && out == "" && passed:
|
||||
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
|
||||
case cause == "" && out == "":
|
||||
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
|
||||
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
|
||||
}
|
||||
if len(cause) > MaxCause {
|
||||
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
|
||||
}
|
||||
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
|
||||
budget := MaxWords
|
||||
if cause != "" {
|
||||
budget -= len(cause) + 1
|
||||
}
|
||||
if len(out) > budget {
|
||||
const marker = "... "
|
||||
tail := out[len(out)-(budget-len(marker)):]
|
||||
out := strings.TrimSpace(f.Output)
|
||||
if len(out) > MaxWords {
|
||||
out = out[len(out)-MaxWords:]
|
||||
// Start on a whole line, and a whole character.
|
||||
if i := strings.IndexByte(tail, '\n'); i >= 0 {
|
||||
tail = strings.TrimLeft(tail[i:], "\n")
|
||||
if i := strings.IndexByte(out, '\n'); i >= 0 {
|
||||
out = "... " + strings.TrimLeft(out[i:], "\n")
|
||||
}
|
||||
out = marker + strings.ToValidUTF8(tail, "")
|
||||
out = strings.ToValidUTF8(out, "")
|
||||
}
|
||||
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
|
||||
out = strings.TrimRight(cause+"\n"+out, "\n")
|
||||
if c := strings.TrimSpace(f.Cause); c != "" && !strings.HasPrefix(firstLine(out), c) {
|
||||
out = c + "\n" + out
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
|
||||
// push the words past MaxWords).
|
||||
const MaxCause = 1024
|
||||
|
||||
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
|
||||
// whether a "PASS" was among them: the package's tests passed before it failed.
|
||||
func dropSummary(out string) (string, bool) {
|
||||
passed := false
|
||||
lines := strings.Split(out, "\n")
|
||||
kept := lines[:0]
|
||||
for _, l := range lines {
|
||||
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
|
||||
switch t := strings.TrimRight(l, "\r"); {
|
||||
case t == "PASS":
|
||||
passed = true
|
||||
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
|
||||
default:
|
||||
kept = append(kept, l)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(kept, "\n")), passed
|
||||
}
|
||||
|
||||
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
|
||||
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
|
||||
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
|
||||
// line holding the same words is not the cause (review round 1).
|
||||
func causeIn(out string) string {
|
||||
lines := strings.Split(out, "\n")
|
||||
for i, l := range lines {
|
||||
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
|
||||
continue
|
||||
}
|
||||
words := "WARNING: DATA RACE"
|
||||
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
|
||||
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
|
||||
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
|
||||
words += " in " + strings.TrimSpace(lines[i+2])
|
||||
}
|
||||
}
|
||||
return words
|
||||
}
|
||||
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
|
||||
for _, l := range lines {
|
||||
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
|
||||
return l
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// firstLine is the first non-empty line of a failure's words.
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
@@ -633,23 +609,64 @@ func firstLine(s string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// failure turns a level that did not pass into a verdict naming what failed first.
|
||||
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
|
||||
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
|
||||
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
|
||||
const MaxFailureLine = 300
|
||||
|
||||
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
|
||||
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
|
||||
// saying how many more (issue 1921).
|
||||
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
|
||||
v := delivery.Verdict{State: delivery.VerdictFail}
|
||||
switch {
|
||||
case len(res.Failed) > 0:
|
||||
f := res.Failed[0]
|
||||
name := failedName(f)
|
||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||
name = res.Module
|
||||
var all []delivery.FailedFirst
|
||||
for _, res := range results {
|
||||
if len(res.Failed) == 0 {
|
||||
why := res.Why
|
||||
if why == "" {
|
||||
why = "the " + level + " level did not pass"
|
||||
}
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
|
||||
continue
|
||||
}
|
||||
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
|
||||
default:
|
||||
why := res.Why
|
||||
if why == "" {
|
||||
why = "the " + level + " level did not pass"
|
||||
for _, f := range res.Failed {
|
||||
name := failedName(f)
|
||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||
name = res.Module
|
||||
}
|
||||
step := level
|
||||
if f.Step != "" {
|
||||
step = f.Step // the lint step's finding, not the level's (ADR 0542 §2)
|
||||
}
|
||||
all = append(all, delivery.FailedFirst{Test: step + ": " + name, Words: failureWords(f)})
|
||||
}
|
||||
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
|
||||
}
|
||||
if len(all) == 0 { // no result: a level said failed with nothing to name
|
||||
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
|
||||
}
|
||||
v.Failed = all[0]
|
||||
v.Failures = failureLines(all)
|
||||
return &v
|
||||
}
|
||||
|
||||
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
|
||||
// and then one entry saying how many more failed.
|
||||
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
|
||||
n := min(len(all), delivery.MaxFailures)
|
||||
out := make([]delivery.FailedFirst, 0, n+1)
|
||||
for _, f := range all[:n] {
|
||||
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
|
||||
}
|
||||
if more := len(all) - n; more > 0 {
|
||||
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
|
||||
func clipLine(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
s = strings.ToValidUTF8(s[:n-3], "")
|
||||
return s + "..."
|
||||
}
|
||||
|
||||
@@ -177,8 +177,8 @@ func TestACandidateThatDoesNotCompileFailsTheCheckWithTheCompilersWords(t *testi
|
||||
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") {
|
||||
t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict)
|
||||
}
|
||||
if n := len(b.steps.tool("test")); n != 1 {
|
||||
t.Errorf("%d test steps, want the first one's failure to end the check", n)
|
||||
if n := len(b.steps.toolAt("test", "unit")); n != 1 {
|
||||
t.Errorf("%d unit test steps, want the first one's failure to end the level", n)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -49,6 +49,34 @@ func firstFailed(v delivery.Verdict) string {
|
||||
return firstWordsLine(f.Words)
|
||||
}
|
||||
|
||||
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
|
||||
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
|
||||
func failedLines(v delivery.Verdict) []string {
|
||||
if len(v.Failures) == 0 {
|
||||
if f := firstFailed(v); f != "" {
|
||||
return []string{f}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
lines := make([]string, 0, len(v.Failures))
|
||||
for _, f := range v.Failures {
|
||||
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
|
||||
func failedCount(v delivery.Verdict) int {
|
||||
n := len(v.Failures)
|
||||
if n > delivery.MaxFailures {
|
||||
var more int
|
||||
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
|
||||
return delivery.MaxFailures + more
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
|
||||
func firstWordsLine(w string) string { return firstLine(w) }
|
||||
|
||||
@@ -155,6 +183,9 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
||||
pc.head = "pass " + commits
|
||||
default:
|
||||
pc.head = "failed: " + firstFailed(v)
|
||||
if n := failedCount(v); n > 1 {
|
||||
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
|
||||
}
|
||||
pc.commits = commits
|
||||
}
|
||||
var counts []string
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// What the build module before this fix sent for two packages that failed after their tests passed (issue 1901, its third
|
||||
// meeting: a try of mesh-controller kept "PASS" as its failure's words). Each is that build module's Output for the real
|
||||
// `go test -json` of a throwaway package (mesh-catalog modules/mesh-build-go testdata/captured/go-test-json-pkg-*.jsonl):
|
||||
// TestMain panicking in a goroutine after m.Run, TestMain exiting 2 after m.Run, and TestMain racing after m.Run.
|
||||
const (
|
||||
oldPassThenPanic = "PASS\npanic: the bus closed after the tests: nats: connection closed\ngoroutine 20 [running]:\n" +
|
||||
"example.com/passthenfail.TestMain.func1()\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||
"created by example.com/passthenfail.TestMain in goroutine 1\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||
"FAIL\texample.com/passthenfail\t0.005s"
|
||||
oldPassThenExit = "PASS\nFAIL\texample.com/mainexit\t0.004s"
|
||||
// TestMain racing after m.Run passed, under go test -race (go-test-json-pkg-racemain.jsonl).
|
||||
oldPassThenRace = "PASS\n==================\nWARNING: DATA RACE\nWrite at 0x000000836428 by goroutine 9:\n example.com/racemain.TestMain.func1()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x24\nPrevious write at 0x000000836428 by main goroutine:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:16 +0x44\n main.main()\n _testmain.go:48 +0x171\nGoroutine 9 (running) created at:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x37\n main.main()\n _testmain.go:48 +0x171\n==================\nFound 1 data race(s)\nFAIL\texample.com/racemain\t1.031s"
|
||||
)
|
||||
|
||||
// A failure with no cause told apart never has "PASS" or "ok" as its words: the checker takes the panic line from the
|
||||
// output, or says that the tests passed and the program failed after them (1901/AC5).
|
||||
func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
|
||||
cases := []struct {
|
||||
name, output, want string
|
||||
}{
|
||||
{"panic after the tests", oldPassThenPanic, "panic: the bus closed after the tests: nats: connection closed"},
|
||||
{"exit after the tests", oldPassThenExit, "the tests passed, then the test program exited with a failure"},
|
||||
{"race after the tests", oldPassThenRace, "WARNING: DATA RACE: Write at 0x000000836428 by goroutine 9 in example.com/racemain.TestMain.func1()"},
|
||||
{"a panic deep in a long output", "PASS\npanic: the seat row vanished\n" + strings.Repeat("goroutine 7 [chan receive]:\n\tsome/frame.go:1 +0x1\n", 300) + "FAIL\tp\t2.0s", "panic: the seat row vanished"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
|
||||
for _, l := range strings.Split(v.Failed.Words, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "PASS" || strings.HasPrefix(l, "ok ") || strings.HasPrefix(l, "ok\t") {
|
||||
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(v.Failed.Words, c.want) {
|
||||
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
|
||||
}
|
||||
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
|
||||
t.Errorf("1901/AC5: status = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A cause the build module told apart is cut to a line's bound, so the words keep their bound (the 1901 review's note).
|
||||
func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
|
||||
f := Failure{Package: "p", Test: "TestX", Cause: strings.Repeat("c", 5000), Output: "x_test.go:1: something"}
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||
if len(v.Failed.Words) > MaxWords+200 {
|
||||
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
|
||||
}
|
||||
}
|
||||
|
||||
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as only go test's last line, with no
|
||||
// cause: the words then say so, never nothing (review round 1). Each output is what that build module sends for the
|
||||
// capture of the same name (go-test-json-pkg-timeout, -exiting, -killed): the package's own output, its "FAIL" line.
|
||||
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
|
||||
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
|
||||
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
|
||||
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
|
||||
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A line of the test's own that starts as go's summary does, or that holds a runtime's words indented, is not taken for
|
||||
// either (review round 1): go prints its summary in exact forms and the runtime's lines at column 0.
|
||||
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
|
||||
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
|
||||
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
|
||||
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
|
||||
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
|
||||
}
|
||||
}
|
||||
|
||||
// The words keep MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
|
||||
func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
|
||||
for _, cause := range []string{"", "a cause", strings.Repeat("c", 3000)} {
|
||||
for _, n := range []int{100, 170, 171, 172, 500} {
|
||||
f := Failure{Package: "p", Test: "TestX", Cause: cause, Output: strings.Repeat("a log line of twenty\n", n) + "the end"}
|
||||
if w := failureWords(f); len(w) > MaxWords {
|
||||
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), MaxWords)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,8 +118,10 @@ func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sameBranchPin names the branch of the tried repository that points at the tried head and that mesh-sdk also has, at the
|
||||
// commit the pin names; empty when there is none.
|
||||
// sameBranchPin names the mesh-sdk branch whose head the pin is, for a try: the branch of the tried repository that points
|
||||
// at the tried head and that mesh-sdk also has, at the commit the pin names, first; else any branch of mesh-sdk but its
|
||||
// trunk whose head the pin is, for a try asked by a bare commit that no branch of the tried repository points at (issue
|
||||
// 1624; try speed S12). Empty when there is none.
|
||||
func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) {
|
||||
mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository)
|
||||
if err != nil {
|
||||
@@ -131,9 +133,6 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
|
||||
same = append(same, name)
|
||||
}
|
||||
}
|
||||
if len(same) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -144,6 +143,16 @@ func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (stri
|
||||
return name, nil
|
||||
}
|
||||
}
|
||||
var any []string
|
||||
for name, h := range sdk {
|
||||
if name != "main" && strings.HasPrefix(h, pin) {
|
||||
any = append(any, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(any)
|
||||
if len(any) > 0 {
|
||||
return any[0], nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
|
||||
@@ -408,3 +408,22 @@ func TestAMergeCheckWithTheSameBranchPinIsStillRefusedWithoutAGroupPullRequest(t
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1624 (met again on 2026-10-25; try speed S12): a try asked by a bare commit, which no branch of the tried
|
||||
// repository points at, still takes a pin that is the head of a mesh-sdk branch, naming that branch; a pin that is the head
|
||||
// of no mesh-sdk branch is still refused.
|
||||
func TestATryByCommitTakesAPinThatIsTheHeadOfAnSDKBranch(t *testing.T) {
|
||||
b, c := sameBranchBench(t, head2[:12])
|
||||
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA, "feat/r6": trunkA2} // the branch moved on
|
||||
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictPass {
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
if n := c.Notes(); len(n) != 1 || n[0] != "pin accepted: head of mesh-sdk feat/r6" {
|
||||
t.Errorf("notes = %q", n)
|
||||
}
|
||||
b, c = sameBranchBench(t, "9aaa5c80ac5c")
|
||||
b.ex.Forge.(*fakeForge).branches[repoA] = map[string]string{"main": trunkA}
|
||||
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "neither on mesh-sdk's trunk") {
|
||||
t.Errorf("a pin no mesh-sdk branch holds: verdict = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,46 +89,37 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Verdicts are the verdicts the checker holds, by key; with a group only that group's. The keys are narrowed by the group
|
||||
// before any value is read, at most MaxReads are read (sixteen at a time), and cut says when it stopped there.
|
||||
func (r Reader) Verdicts(group string) (out []VerdictRead, cut string, err error) {
|
||||
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
|
||||
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
|
||||
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
|
||||
return nil, "", fmt.Errorf("the verdicts state cannot be read here")
|
||||
return nil, fmt.Errorf("the verdicts state cannot be read here")
|
||||
}
|
||||
keys, err := r.VerdictState.Keys()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var matching []string
|
||||
var out []VerdictRead
|
||||
for _, k := range keys {
|
||||
if group != "" {
|
||||
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
|
||||
continue
|
||||
}
|
||||
}
|
||||
matching = append(matching, k)
|
||||
}
|
||||
matching, cut = capKeys(matching, "give a group to narrow the answer")
|
||||
rows := make([]VerdictRead, len(matching))
|
||||
found := make([]bool, len(matching))
|
||||
if err := parallel(len(matching), func(i int) error {
|
||||
raw, err := r.VerdictState.Get(matching[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return err
|
||||
raw, err := r.VerdictState.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows[i], err = r.read(matching[i], raw)
|
||||
found[i] = err == nil
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
for i, row := range rows {
|
||||
if found[i] {
|
||||
out = append(out, row)
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
vr, err := r.read(k, raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, vr)
|
||||
}
|
||||
return out, cut, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
|
||||
@@ -142,95 +133,74 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
|
||||
for from := 0; from < len(keys); from += ReadWorkers * 4 {
|
||||
batch := keys[from:min(from+ReadWorkers*4, len(keys))]
|
||||
cands := make([]delivery.Candidate, len(batch))
|
||||
ok := make([]bool, len(batch))
|
||||
_ = parallel(len(batch), func(i int) error {
|
||||
raw, err := r.CandidateState.Get(batch[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
cands[i], ok[i] = c, err == nil
|
||||
return nil
|
||||
})
|
||||
for i, c := range cands {
|
||||
if !ok[i] {
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidateState.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository != repository || p.Number != number {
|
||||
continue
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Repository != repository || p.Number != number {
|
||||
continue
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
vraw, err := r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
if len(vraw) == 0 {
|
||||
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
||||
}
|
||||
return r.read(vkey, vraw)
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
vraw, err := r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
if len(vraw) == 0 {
|
||||
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
|
||||
}
|
||||
return r.read(vkey, vraw)
|
||||
}
|
||||
}
|
||||
if cut != "" {
|
||||
return VerdictRead{}, fmt.Errorf("no candidate among the first %d holds %s#%d", MaxReads, repository, number)
|
||||
}
|
||||
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
|
||||
}
|
||||
|
||||
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
|
||||
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, string, error) {
|
||||
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
|
||||
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
|
||||
return nil, "", fmt.Errorf("the kept state cannot be read here")
|
||||
return nil, fmt.Errorf("the kept state cannot be read here")
|
||||
}
|
||||
if digest != "" {
|
||||
key, err := delivery.KeptKey(level, digest)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
raw, err := r.KeptState.Get(key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, "", fmt.Errorf("no kept verdict %s", key)
|
||||
return nil, fmt.Errorf("no kept verdict %s", key)
|
||||
}
|
||||
return map[string]json.RawMessage{key: raw}, "", nil
|
||||
return map[string]json.RawMessage{key: raw}, nil
|
||||
}
|
||||
keys, err := r.KeptState.Keys()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var matching []string
|
||||
for _, k := range keys {
|
||||
if l, _, err := delivery.ParseKeptKey(k); err == nil && (level == "" || l == level) {
|
||||
matching = append(matching, k)
|
||||
}
|
||||
}
|
||||
matching, cut := capKeys(matching, "give a level and a digest to read one")
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
if err := parallel(len(matching), func(i int) (err error) {
|
||||
values[i], err = r.KeptState.Get(matching[i])
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
out := map[string]json.RawMessage{}
|
||||
for i, k := range matching {
|
||||
if len(values[i]) > 0 {
|
||||
out[k] = values[i]
|
||||
for _, k := range keys {
|
||||
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
|
||||
continue
|
||||
}
|
||||
raw, err := r.KeptState.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out[k] = raw
|
||||
}
|
||||
}
|
||||
return out, cut, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PreviewRead is one preview as stored, with the parts of its key.
|
||||
@@ -244,18 +214,17 @@ type PreviewRead struct {
|
||||
|
||||
// Previews are the previews the checker holds, each as stored. With a repository only that repository's; with a head (a
|
||||
// prefix of at least 7 characters is enough) only previews of that head. A key that is no preview key is listed as unreadable.
|
||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, string, error) {
|
||||
func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, error) {
|
||||
if r.PreviewState.Keys == nil || r.PreviewState.Get == nil {
|
||||
return nil, nil, "", fmt.Errorf("the previews state cannot be read here")
|
||||
return nil, nil, fmt.Errorf("the previews state cannot be read here")
|
||||
}
|
||||
keys, err := r.PreviewState.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, "", err
|
||||
return nil, nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
type part struct{ repo, head, trunk string }
|
||||
parts := map[string]part{}
|
||||
var matching, bad []string
|
||||
out := []PreviewRead{}
|
||||
var bad []string
|
||||
for _, k := range keys {
|
||||
repo, h, trunk, err := delivery.ParsePreviewKey(k)
|
||||
if err != nil {
|
||||
@@ -265,25 +234,15 @@ func (r Reader) Previews(repository, head string) ([]PreviewRead, []string, stri
|
||||
if (repository != "" && repo != repository) || (head != "" && !strings.HasPrefix(h, head)) {
|
||||
continue
|
||||
}
|
||||
parts[k] = part{repo, h, trunk}
|
||||
matching = append(matching, k)
|
||||
}
|
||||
matching, cut := capKeys(matching, "give a repository and a head to narrow the answer")
|
||||
values := make([]json.RawMessage, len(matching))
|
||||
if err := parallel(len(matching), func(i int) (err error) {
|
||||
values[i], err = r.PreviewState.Get(matching[i])
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
out := []PreviewRead{}
|
||||
for i, k := range matching {
|
||||
if len(values[i]) > 0 {
|
||||
p := parts[k]
|
||||
out = append(out, PreviewRead{Key: k, Repository: p.repo, Head: p.head, Trunk: p.trunk, Preview: values[i]})
|
||||
raw, err := r.PreviewState.Get(k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
out = append(out, PreviewRead{Key: k, Repository: repo, Head: h, Trunk: trunk, Preview: raw})
|
||||
}
|
||||
}
|
||||
return out, bad, cut, nil
|
||||
return out, bad, nil
|
||||
}
|
||||
|
||||
// RunningStep is a step in flight on a build module.
|
||||
@@ -341,28 +300,16 @@ func (r Reader) Status() (Status, error) {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
|
||||
continue
|
||||
}
|
||||
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
|
||||
newestFirst(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
|
||||
}
|
||||
steps := make([]delivery.Step, len(keys))
|
||||
isRunning := make([]bool, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := sr.Get(keys[i])
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
raw, err := sr.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
|
||||
return nil
|
||||
})
|
||||
for i := range keys {
|
||||
if !isRunning[i] {
|
||||
continue
|
||||
}
|
||||
step := steps[i]
|
||||
step, err := delivery.DecodeStep(raw)
|
||||
if err != nil || step.State != delivery.StepRunning {
|
||||
continue
|
||||
}
|
||||
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
|
||||
if !step.Started.IsZero() {
|
||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||
@@ -397,41 +344,23 @@ func (r Reader) Status() (Status, error) {
|
||||
return st, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
total := len(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
if cut != "" {
|
||||
st.Notes = append(st.Notes, fmt.Sprintf("read the first %d candidates of %d", MaxReads, total))
|
||||
}
|
||||
type judged struct {
|
||||
c delivery.Candidate
|
||||
raw json.RawMessage
|
||||
ok bool
|
||||
}
|
||||
rows := make([]judged, len(keys))
|
||||
_ = parallel(len(keys), func(i int) error {
|
||||
raw, err := r.CandidateState.Get(keys[i])
|
||||
for _, k := range keys {
|
||||
raw, err := r.CandidateState.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
rows[i].c = c
|
||||
if vraw, err := r.VerdictState.Get(vkey); err == nil {
|
||||
rows[i].raw, rows[i].ok = vraw, true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
for _, row := range rows {
|
||||
if !row.ok {
|
||||
continue
|
||||
}
|
||||
c, raw := row.c, row.raw
|
||||
raw, err = r.VerdictState.Get(vkey)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
st.Unjudged = append(st.Unjudged, c.Group)
|
||||
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||
|
||||
@@ -3,9 +3,7 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -53,7 +51,7 @@ func readerOf(t *testing.T) (Reader, string) {
|
||||
|
||||
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
|
||||
rd, key := readerOf(t)
|
||||
vs, _, err := rd.Verdicts("")
|
||||
vs, err := rd.Verdicts("")
|
||||
if err != nil || len(vs) != 1 || vs[0].Key != key {
|
||||
t.Fatalf("verdicts = %+v, %v", vs, err)
|
||||
}
|
||||
@@ -63,7 +61,7 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
|
||||
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
|
||||
t.Errorf("reports = %v", vs[0].Reports)
|
||||
}
|
||||
if only, _, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||
if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
|
||||
t.Errorf("another group's verdicts = %+v", only)
|
||||
}
|
||||
}
|
||||
@@ -87,11 +85,11 @@ func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
|
||||
|
||||
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
|
||||
rd, _ := readerOf(t)
|
||||
all, _, err := rd.Kept("", "")
|
||||
all, err := rd.Kept("", "")
|
||||
if err != nil || len(all) != 1 {
|
||||
t.Fatalf("kept = %v, %v", all, err)
|
||||
}
|
||||
one, _, err := rd.Kept("unit", digA)
|
||||
one, err := rd.Kept("unit", digA)
|
||||
if err != nil || len(one) != 1 {
|
||||
t.Fatalf("one kept = %v, %v", one, err)
|
||||
}
|
||||
@@ -100,10 +98,10 @@ func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { //
|
||||
t.Errorf("kept value = %+v, %v", k, err)
|
||||
}
|
||||
}
|
||||
if none, _, _ := rd.Kept("integration", ""); len(none) != 0 {
|
||||
if none, _ := rd.Kept("integration", ""); len(none) != 0 {
|
||||
t.Errorf("integration = %v", none)
|
||||
}
|
||||
if _, _, err := rd.Kept("unit", digB); err == nil {
|
||||
if _, err := rd.Kept("unit", digB); err == nil {
|
||||
t.Error("a digest nothing kept was answered")
|
||||
}
|
||||
}
|
||||
@@ -117,17 +115,17 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
|
||||
}
|
||||
key, _ := delivery.PreviewKey(repoA, head1, head2)
|
||||
rd := Reader{PreviewState: fakeState{key: data, "junk": []byte("{}")}.read()}
|
||||
got, bad, _, err := rd.Previews(repoA, head1[:8])
|
||||
got, bad, err := rd.Previews(repoA, head1[:8])
|
||||
if err != nil || len(got) != 1 || got[0].Key != key || got[0].Head != head1 || got[0].Trunk != head2 || string(got[0].Preview) != string(data) {
|
||||
t.Fatalf("previews = %+v, %v", got, err)
|
||||
}
|
||||
if len(bad) != 1 || bad[0] != "junk" {
|
||||
t.Errorf("unreadable = %v", bad)
|
||||
}
|
||||
if got, _, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||
if got, _, _ := rd.Previews("novox/other", ""); len(got) != 0 {
|
||||
t.Errorf("another repository's = %+v", got)
|
||||
}
|
||||
if _, _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||
if _, _, err := (Reader{}).Previews("", ""); err == nil {
|
||||
t.Error("answered with no state to read")
|
||||
}
|
||||
}
|
||||
@@ -235,121 +233,3 @@ func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
||||
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||
}
|
||||
}
|
||||
|
||||
// slowBucket wraps a state with a Get that takes delay and counts the values read, the lists and the most Gets in flight.
|
||||
type slowBucket struct {
|
||||
vals fakeState
|
||||
reads, lists atomic.Int64
|
||||
inFlight, maxFlight atomic.Int64
|
||||
}
|
||||
|
||||
func (b *slowBucket) get(key string) (json.RawMessage, error) {
|
||||
b.reads.Add(1)
|
||||
now := b.inFlight.Add(1)
|
||||
for m := b.maxFlight.Load(); now > m && !b.maxFlight.CompareAndSwap(m, now); m = b.maxFlight.Load() {
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
b.inFlight.Add(-1)
|
||||
return b.vals.read().Get(key)
|
||||
}
|
||||
|
||||
func (b *slowBucket) read() StateRead {
|
||||
return StateRead{Keys: func() ([]string, error) { b.lists.Add(1); return b.vals.read().Keys() }, Get: b.get}
|
||||
}
|
||||
|
||||
func (b *slowBucket) Keys() ([]string, error) { return b.read().Keys() }
|
||||
func (b *slowBucket) Get(key string) (json.RawMessage, error) { return b.get(key) }
|
||||
func (b *slowBucket) Create(string, []byte) error { return nil }
|
||||
|
||||
// 1827: a read tool lists the keys of a bucket of thousands and reads only the values its answer needs, sixteen at a time, at
|
||||
// most MaxReads, saying when it cut (on the live shape of 2026-10-26: kept 7990 keys, steps 10034, tries 1000s).
|
||||
func TestReadToolsReadOnlyWhatTheyNeedInParallelAndSayWhenCut(t *testing.T) { // 1827
|
||||
kept := fakeState{}
|
||||
for i := 0; i < 1800; i++ {
|
||||
level := map[bool]string{true: "unit", false: "integration"}[i%9 != 0]
|
||||
k, _ := delivery.KeptKey(level, fmt.Sprintf("%064x", i))
|
||||
kept[k] = []byte(`{"v":1}`)
|
||||
}
|
||||
previews := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
repo := map[bool]string{true: repoA, false: repoSDK}[i < 300]
|
||||
k, _ := delivery.PreviewKey(repo, fmt.Sprintf("%08x%032x", i, 0), trunkA)
|
||||
previews[k] = []byte(`{"v":1}`)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
call func(*slowBucket) (rows int, cut string, err error)
|
||||
rows int
|
||||
reads, lists int64
|
||||
cut bool
|
||||
}{
|
||||
{"kept of one key", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("unit", fmt.Sprintf("%064x", 1))
|
||||
return len(m), c, err
|
||||
}, 1, 1, 0, false},
|
||||
{"kept of a level", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("integration", "")
|
||||
return len(m), c, err
|
||||
}, 200, 200, 1, false},
|
||||
{"kept of everything", func(b *slowBucket) (int, string, error) {
|
||||
m, c, err := Reader{KeptState: b.read()}.Kept("", "")
|
||||
return len(m), c, err
|
||||
}, MaxReads, MaxReads, 1, true},
|
||||
{"previews of a repository", func(b *slowBucket) (int, string, error) {
|
||||
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoSDK, "")
|
||||
return len(p), c, err
|
||||
}, 1200, 1000, 1, true},
|
||||
{"previews of a repository and a head", func(b *slowBucket) (int, string, error) {
|
||||
p, _, c, err := Reader{PreviewState: b.read()}.Previews(repoA, fmt.Sprintf("%08x", 7))
|
||||
return len(p), c, err
|
||||
}, 1, 1, 1, false},
|
||||
} {
|
||||
b := &slowBucket{}
|
||||
b.vals = kept
|
||||
if strings.HasPrefix(row.name, "previews") {
|
||||
b.vals = previews
|
||||
}
|
||||
start := time.Now()
|
||||
rows, cut, err := row.call(b)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", row.name, err)
|
||||
}
|
||||
if rows != min(row.rows, MaxReads) || b.reads.Load() != row.reads || b.lists.Load() != row.lists || (cut != "") != row.cut {
|
||||
t.Errorf("%s: %d rows, %d reads, %d lists, cut %q; want %d, %d, %d, cut %v", row.name, rows, b.reads.Load(), b.lists.Load(), cut, min(row.rows, MaxReads), row.reads, row.lists, row.cut)
|
||||
}
|
||||
if row.reads > 20 && b.maxFlight.Load() < 2 {
|
||||
t.Errorf("%s: values were read one at a time", row.name)
|
||||
}
|
||||
t.Logf("%s: %d values in %v (one after the other: %v)", row.name, b.reads.Load(), time.Since(start), time.Duration(b.reads.Load())*time.Millisecond)
|
||||
}
|
||||
|
||||
// The status reads the newest steps of a bucket of 1500 first, at most MaxReads, and finds the one that runs.
|
||||
now := time.Date(2026, 10, 26, 7, 0, 0, 0, time.UTC)
|
||||
steps := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
id := fmt.Sprintf("mvo5%04dabcdef", i)
|
||||
step := delivery.Step{ID: id, Tool: "t", Node: "g14", State: delivery.StepPassed}
|
||||
if i == 1499 {
|
||||
step.State, step.Started = delivery.StepRunning, now.Add(-time.Minute)
|
||||
}
|
||||
raw, _ := delivery.EncodeStep(step)
|
||||
steps["g14."+id] = raw
|
||||
}
|
||||
sb := &slowBucket{vals: steps}
|
||||
st, err := Reader{StepStates: map[string]StateRead{"mesh-build-go": sb.read()}, HaveState: fakeState{"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||
Now: func() time.Time { return now }}.Status()
|
||||
if err != nil || sb.reads.Load() != MaxReads || len(st.Running) != 1 || sb.maxFlight.Load() < 2 {
|
||||
t.Errorf("status: %d reads (want %d), running %+v, in flight %d, %v", sb.reads.Load(), MaxReads, st.Running, sb.maxFlight.Load(), err)
|
||||
}
|
||||
|
||||
// A new try reads the newest TryScan of 1500 ended tries, not all, and still finds the same head asked twice.
|
||||
tries := fakeState{}
|
||||
for i := 0; i < 1500; i++ {
|
||||
raw, _ := encodeTry(Try{ID: fmt.Sprintf("try-%d", 1700000000000000000+i), Repository: repoA, Head: head1, Trunk: trunkA, State: TryPass})
|
||||
tries[fmt.Sprintf("try-%d", 1700000000000000000+i)] = raw
|
||||
}
|
||||
tb := &slowBucket{vals: tries}
|
||||
if _, err := AskTry(tb, tryTrunks, TryAsk{Repository: repoA, Head: head2, Trunk: trunkA}, now); err != nil || tb.reads.Load() != TryScan {
|
||||
t.Errorf("AskTry read %d of 1500 tries (want %d), %v", tb.reads.Load(), TryScan, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,9 @@ type Failure struct {
|
||||
Output string `json:"output,omitempty"`
|
||||
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
|
||||
Cause string `json:"cause,omitempty"`
|
||||
// Step is "lint" for a finding of the lint step (gofmt, go vet), which runs beside the unit level and is reported as
|
||||
// its own failure, not the level's (ADR 0542 §2).
|
||||
Step string `json:"step,omitempty"`
|
||||
}
|
||||
|
||||
// TestTime is one test and the seconds it took.
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const repoC = "novox/mesh-controller"
|
||||
|
||||
func statusOf(t *testing.T, r *rig, repo string, n int, head string) string {
|
||||
t.Helper()
|
||||
key, _ := delivery.CheckerOwedKey(repo, n, head)
|
||||
raw, _, found, _ := r.owed.Get(context.Background(), key)
|
||||
if !found {
|
||||
return ""
|
||||
}
|
||||
o, err := delivery.DecodeCheckerOwed(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return o.State
|
||||
}
|
||||
|
||||
// 1791 note 9 (issue 1921, try speed S9): two of a group's three members merge. The candidate keeps them, marked merged,
|
||||
// with the same heads; their trunks moved by their own merges. No new check is asked, and the member left keeps its
|
||||
// passed status, so the merge of the rest is not refused.
|
||||
func TestTwoMembersMergingAskNoNewCheckAndTheThirdKeepsItsPass(t *testing.T) {
|
||||
ctx, j := context.Background(), passJudge()
|
||||
r := newRig(t, j)
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, repoC: trunkA})
|
||||
a, s, c := pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(repoC, 3, head1)
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 || statusOf(t, r, repoC, 3, head1) != "success" {
|
||||
t.Fatalf("judged %d times, the third's status %q", j.count(), statusOf(t, r, repoC, 3, head1))
|
||||
}
|
||||
a.Merged, s.Merged = true, true
|
||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", a, s, c))
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if j.count() != 1 {
|
||||
t.Errorf("a new check was asked after two members merged: judged %d times", j.count())
|
||||
}
|
||||
if got := statusOf(t, r, repoC, 3, head1); got != "success" {
|
||||
t.Errorf("the third member's status is %q, want success", got)
|
||||
}
|
||||
// The trunk of the member left moves: that is a re-judge as before.
|
||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS2, repoC: trunkA2})
|
||||
r.ch.TrunkMoved(ctx)
|
||||
r.ch.Wait()
|
||||
if j.count() != 2 {
|
||||
t.Errorf("the member left's trunk moved and it was judged %d times, want again", j.count())
|
||||
}
|
||||
}
|
||||
|
||||
// 1791 note 8 (issue 1921, try speed S9): a group opened member by member writes a candidate for each partial set. When
|
||||
// the group's candidate moves to new heads, the check of the older heads is stopped and ends failed, superseded, instead
|
||||
// of running on or staying running; it puts no status, as the newer check owns the heads they share.
|
||||
func TestACheckWhoseHeadsTheGroupNoLongerHoldsIsStoppedAndEndsSuperseded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
started := make(chan struct{})
|
||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||
if len(c.Pulls) == 1 {
|
||||
close(started)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Outcome{Verdict: failure("checker", ctx.Err().Error())}, nil
|
||||
case <-time.After(3 * time.Second):
|
||||
return Outcome{Verdict: passing()}, nil // never stopped
|
||||
}
|
||||
}
|
||||
return Outcome{Verdict: passing()}, nil
|
||||
}}
|
||||
r := newRig(t, j)
|
||||
partial := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
whole := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
|
||||
r.ch.Candidate(ctx, "feat%2Fx", partial)
|
||||
<-started
|
||||
r.ch.Candidate(ctx, "feat%2Fx", whole)
|
||||
done := make(chan struct{})
|
||||
go func() { r.ch.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the check of the older heads was not stopped")
|
||||
}
|
||||
old := r.verdict(t, keyOf(t, "feat/x", partial))
|
||||
if old.State != delivery.VerdictFail || !strings.Contains(old.Failed.Words, "superseded") {
|
||||
t.Errorf("the older heads' verdict = %+v, want failed, superseded", old)
|
||||
}
|
||||
if v := r.verdict(t, keyOf(t, "feat/x", whole)); v.State != delivery.VerdictPass {
|
||||
t.Errorf("the newer heads' verdict = %+v", v)
|
||||
}
|
||||
if got := statusOf(t, r, repoA, 7, head1); got != "success" {
|
||||
t.Errorf("the shared head's status is %q, want the newer check's success", got)
|
||||
}
|
||||
}
|
||||
@@ -29,8 +29,6 @@ const (
|
||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||
// MaxTries is how many tries may wait or run at once, mesh-wide: each holds a build node for minutes.
|
||||
MaxTries = 4
|
||||
// TryScan is how many of the newest tries a new ask reads to find the ones that wait or run (issue 1827).
|
||||
TryScan = 64
|
||||
// TryKeep is how long an ended try's key is kept before the checker removes it.
|
||||
TryKeep = 7 * 24 * time.Hour
|
||||
)
|
||||
@@ -69,7 +67,9 @@ type Try struct {
|
||||
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
||||
Said string `json:"said,omitempty"`
|
||||
// Took is the time from its start to its end, as Go writes a duration.
|
||||
Took string `json:"took,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
|
||||
// (issue 1921); or what else failed first (a conflict, a refused step).
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
@@ -157,33 +157,15 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the tries cannot be listed: %w", err)
|
||||
}
|
||||
// An ended try's key stays TryKeep, so the bucket holds thousands; one that waits or runs was asked lately. Only the
|
||||
// newest TryScan keys are read, in parallel (a try id is the time in nanoseconds, so the newest sorts last).
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(keys[i]) != len(keys[j]) {
|
||||
return len(keys[i]) > len(keys[j])
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
if len(keys) > TryScan {
|
||||
keys = keys[:TryScan]
|
||||
}
|
||||
tries := make([]Try, len(keys))
|
||||
live := make([]bool, len(keys))
|
||||
if err := parallel(len(keys), func(i int) error {
|
||||
raw, err := store.Get(keys[i])
|
||||
sort.Strings(keys)
|
||||
var open []string
|
||||
for _, k := range keys {
|
||||
raw, err := store.Get(k)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
cur, err := DecodeTry(raw)
|
||||
tries[i], live[i] = cur, err == nil && !cur.Over()
|
||||
return nil
|
||||
}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
var open []string
|
||||
for i, cur := range tries {
|
||||
if !live[i] {
|
||||
if err != nil || cur.Over() {
|
||||
continue
|
||||
}
|
||||
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
@@ -359,9 +341,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
t.State = TryPass
|
||||
default:
|
||||
t.State = TryFail
|
||||
if f := firstFailed(v); f != "" {
|
||||
t.Failed = []string{f}
|
||||
}
|
||||
t.Failed = failedLines(v)
|
||||
}
|
||||
if t.Tests != "" {
|
||||
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
|
||||
|
||||
@@ -22,8 +22,6 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -190,18 +188,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
call := callerOn(conn, cred.User)
|
||||
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
|
||||
var turn sync.Mutex
|
||||
kv, err := process.EssentialBucket(ctx, js, "bridge", headsBucket, "keep the heads it wrote on the forge stream")
|
||||
kv, err := js.KeyValue(ctx, headsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("the bucket %s: %w", headsBucket, err)
|
||||
}
|
||||
heads := kvHeads{kv}
|
||||
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
|
||||
rec := &recorder{pub: js, heads: heads, logger: logger}
|
||||
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
|
||||
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
|
||||
performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once")
|
||||
performedKV, err := js.KeyValue(ctx, performedBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("the bucket %s: %w", performedBucket, err)
|
||||
}
|
||||
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
|
||||
watching := make(chan error, 3)
|
||||
@@ -209,12 +207,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
bucket string
|
||||
react func(ctx context.Context, key string, rev uint64, value []byte)
|
||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
|
||||
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
|
||||
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
|
||||
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
|
||||
if kv == nil {
|
||||
kv, err := js.KeyValue(ctx, w.bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
// The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again
|
||||
// by its delivery when the gate arrives.
|
||||
logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
|
||||
}
|
||||
go func() {
|
||||
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
@@ -229,8 +231,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
go onReconnect(ctx, again, said, logger)
|
||||
cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
// Essential: the forge's events are what the bridge carries.
|
||||
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, eventsStream, err)
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
hear(&turn, func() {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -13,8 +12,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -69,16 +66,6 @@ func body(m map[string]any) []byte {
|
||||
}
|
||||
|
||||
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
|
||||
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
|
||||
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
|
||||
var said []string
|
||||
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
|
||||
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "bridge", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
|
||||
t.Errorf("an essential bucket refused: %v, want its grant named", err)
|
||||
}
|
||||
})
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
@@ -243,8 +230,3 @@ func someFiles(_, _, path, _ string) ([]byte, error) {
|
||||
}
|
||||
return nil, bridge.ErrNotFound
|
||||
}
|
||||
|
||||
// refusingBus refuses every bucket with err (issue 1945).
|
||||
type refusingBus struct{ err error }
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -7,10 +7,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -440,9 +438,8 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
|
||||
|
||||
// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59,
|
||||
// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the
|
||||
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, its new form
|
||||
// (fingerprint or reason) does not move, and so the second writes no value at all (issue 1758, I9, ADR 0499 check clause 6
|
||||
// in miniature; the operator, 2026-10-25: a module's value moves only with the module).
|
||||
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form
|
||||
// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature).
|
||||
func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json")
|
||||
if err != nil {
|
||||
@@ -451,7 +448,7 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
|
||||
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
|
||||
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {})
|
||||
publish := func(merge, root string, objects map[string]string) {
|
||||
merged := func(merge, root string) delivery.Module {
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root},
|
||||
Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}})
|
||||
@@ -461,8 +458,6 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
at := func(merge string) delivery.Module {
|
||||
var m delivery.Module
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := modules.Get(ctx, "mesh-sdk"); err == nil {
|
||||
@@ -474,35 +469,13 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||
t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge)
|
||||
return m
|
||||
}
|
||||
const firstMerge, secondMerge, thirdMerge = "89abcdef0123456789abcdef0123456789abcdef", "99abcdef0123456789abcdef0123456789abcdef", "a9abcdef0123456789abcdef0123456789abcdef"
|
||||
publish(firstMerge, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", objects)
|
||||
first := at(firstMerge)
|
||||
if first.Inputs2 == nil || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
|
||||
t.Fatalf("kept inputs %+v, want a named tree", first.Inputs2)
|
||||
first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
|
||||
second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
|
||||
if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
|
||||
t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2)
|
||||
}
|
||||
// The go/-only merge, then one that changes src/: the stream is taken in order, so once the third is written the second
|
||||
// was heard, and it wrote nothing: the module's value stays at the commit its fingerprint first appeared at (the
|
||||
// operator, 2026-10-25).
|
||||
publish(secondMerge, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", objects)
|
||||
changed := maps.Clone(objects)
|
||||
changed["src"] = "6666666666666666666666666666666666666666"
|
||||
publish(thirdMerge, "cccccccccccccccccccccccccccccccccccccccc", changed)
|
||||
at(thirdMerge)
|
||||
history, err := modules.History(ctx, "mesh-sdk")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var commits []string
|
||||
for _, e := range history {
|
||||
if m, err := delivery.DecodeModule(e.Value()); err == nil {
|
||||
commits = append(commits, m.Commit)
|
||||
if m.Commit == firstMerge && (m.Inputs2 == nil || m.Inputs2.Tree != first.Inputs2.Tree || m.FP2 != first.FP2 || m.None2 != first.None2) {
|
||||
t.Errorf("the value at the first merge moved: %+v, was %+v", m, first)
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(commits, secondMerge) {
|
||||
t.Errorf("the go/-only merge wrote a value: the commits written are %v", commits)
|
||||
if first.FP2 != second.FP2 || first.None2 != second.None2 {
|
||||
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -642,33 +615,17 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
||||
return got
|
||||
}
|
||||
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
|
||||
held := read(agreed)
|
||||
if !agreed(held) {
|
||||
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
|
||||
if m := read(agreed); !agreed(m) {
|
||||
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", m.FP2, m.None2, m.Toolchains)
|
||||
}
|
||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept
|
||||
// (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if e, err := modules.Get(ctx, "avahi"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
|
||||
t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
|
||||
}
|
||||
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
|
||||
if moved == string(data) {
|
||||
t.Fatal("the merge names no tree to move")
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", []byte(moved)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disagree := func(m delivery.Module) bool {
|
||||
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
|
||||
}
|
||||
if m := read(disagree); !disagree(m) {
|
||||
t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||
t.Fatalf("within 5 s of n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||
}
|
||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -240,21 +240,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
candidates, err := process.EssentialBucket(ctx, js, "indexer", delivery.CandidatesBucket, "write a candidate's fingerprints")
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
candidates, err := bucket(delivery.CandidatesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := process.EssentialBucket(ctx, js, "indexer", delivery.ModulesBucket, "write a module's fingerprint")
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := process.EssentialBucket(ctx, js, "indexer", delivery.ManifestsBucket, "keep a module's manifest")
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := bucket(delivery.BuildsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
|
||||
// such a module waits for its path, every other is indexed (issue 1945).
|
||||
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
|
||||
known := index.NewBuilds()
|
||||
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
|
||||
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
|
||||
@@ -265,14 +273,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
known.Delete(e.Key())
|
||||
}
|
||||
}
|
||||
errs := make(chan error, 1)
|
||||
if builds != nil {
|
||||
if w, err := settle(ctx, builds, each); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.BuildsBucket, err, "a module whose key names no directory is not given one until the indexer starts again")
|
||||
} else {
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
}
|
||||
w, err := settle(ctx, builds, each)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errs := make(chan error, 1)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
|
||||
// The indexer asks the forge for the objects of a manifest's named paths as its own account (ADR 0499 clause 3).
|
||||
account, err := process.ReadAccount(path)
|
||||
@@ -290,17 +296,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
sourceErrs := make(chan error, 1)
|
||||
if source := process.OptionalBucket(ctx, js, delivery.SourceBucket, "no source revision is read until the indexer starts again", logger.Printf); source != nil {
|
||||
switch source, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
default:
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
|
||||
}
|
||||
}
|
||||
if sw, err := settle(ctx, source, each); err != nil {
|
||||
process.NotWatched(logger.Printf, delivery.SourceBucket, err, "no source revision is read until the indexer starts again")
|
||||
} else {
|
||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||
sw, err := settle(ctx, source, each)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||
}
|
||||
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||
@@ -370,8 +381,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
// Essential: the forge's merges are what the indexer indexes.
|
||||
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { handle(ctx, m, ix, logger) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
|
||||
@@ -3,7 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -55,14 +55,11 @@ func (w *previewWatch) ensure() error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
|
||||
return nil
|
||||
case err != nil:
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", delivery.PreviewsBucket, err)
|
||||
return nil
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
process.NotWatched(w.logger.Printf, delivery.PreviewsBucket, err, "it is watched when the checker says it wrote a preview")
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
w.watched = true
|
||||
go func() {
|
||||
|
||||
@@ -28,11 +28,8 @@ import (
|
||||
//
|
||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||
type toolchainGate struct {
|
||||
@@ -41,14 +38,14 @@ type toolchainGate struct {
|
||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||
logf func(format string, a ...any)
|
||||
known bool // the assignments are settled
|
||||
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||
standing map[string]bool // <node>.<module> with an open assignment
|
||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||
}
|
||||
|
||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||
for _, m := range modules {
|
||||
g.modules[m] = true
|
||||
}
|
||||
@@ -70,31 +67,20 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
||||
return g.pass(module, key)
|
||||
}
|
||||
|
||||
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||
// whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||
node, module, ok := strings.Cut(key, ".")
|
||||
if !ok || !g.modules[module] {
|
||||
return false
|
||||
}
|
||||
assigned, open := openAssignment(value)
|
||||
open = open && !deleted
|
||||
open := !deleted && openAssignment(value)
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
was, stood := g.standing[key]
|
||||
since := assigned
|
||||
if since.IsZero() {
|
||||
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||
since = at
|
||||
if stood {
|
||||
since = was
|
||||
}
|
||||
}
|
||||
if open == stood && (!open || since.Equal(was)) {
|
||||
if g.standing[key] == open {
|
||||
return false
|
||||
}
|
||||
if open {
|
||||
g.standing[key] = since
|
||||
g.standing[key] = true
|
||||
} else {
|
||||
delete(g.standing, key)
|
||||
}
|
||||
@@ -125,58 +111,37 @@ func (g *toolchainGate) Settled() bool {
|
||||
return changed
|
||||
}
|
||||
|
||||
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||
// known), deleted otherwise. The caller holds mu.
|
||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||
// otherwise. The caller holds mu.
|
||||
func (g *toolchainGate) pass(module, key string) bool {
|
||||
value, held := g.raw[module][key]
|
||||
if !held {
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
node, _, err := delivery.ParseToolchainKey(key)
|
||||
why := ""
|
||||
if err == nil && g.known {
|
||||
why = g.leftOut(module, node, value)
|
||||
}
|
||||
if why == "" {
|
||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||
delete(g.left, module+"/"+key)
|
||||
return g.forward(module, key, value, false)
|
||||
}
|
||||
if g.left[module+"/"+key] != why {
|
||||
g.left[module+"/"+key] = why
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||
if !g.left[module+"/"+key] {
|
||||
g.left[module+"/"+key] = true
|
||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||
delivery.ToolchainsBucket(module), key, module, node)
|
||||
}
|
||||
return g.forward(module, key, nil, true)
|
||||
}
|
||||
|
||||
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||
// The caller holds mu.
|
||||
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||
since, open := g.standing[node+"."+module]
|
||||
if !open {
|
||||
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err == nil && tc.At.Before(since) {
|
||||
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||
// it was assigned, zero where the value does not say.
|
||||
func openAssignment(value []byte) (time.Time, bool) {
|
||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||
func openAssignment(value []byte) bool {
|
||||
var a struct {
|
||||
Assigned time.Time `json:"assigned"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
Ended time.Time `json:"ended"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(value, &a) != nil {
|
||||
return time.Time{}, false
|
||||
return false
|
||||
}
|
||||
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||
return a.Ended.IsZero() && a.State == ""
|
||||
}
|
||||
|
||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||
@@ -190,19 +155,17 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||
w, err := settle(ctx, kv, each)
|
||||
if err != nil {
|
||||
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
|
||||
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||
return errs, nil
|
||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||
}
|
||||
if gate.Settled() {
|
||||
changed(ctx)
|
||||
}
|
||||
go func() {
|
||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||
changed(ctx)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -45,10 +44,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||
}
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||
if !gate.Settled() {
|
||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||
}
|
||||
@@ -64,20 +63,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
t.Fatal("an unassigned node's report changed what was passed on")
|
||||
}
|
||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||
t.Fatal("assigning ace again passed nothing on")
|
||||
}
|
||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||
}
|
||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||
t.Fatal("a pending assignment is no assignment on the machine")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||
t.Fatalf("got %q", v)
|
||||
}
|
||||
@@ -91,75 +90,23 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||
key, _ := gateReport(t, node, version)
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value
|
||||
set := &buildrun.ToolchainSet{}
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||
gate.Settled()
|
||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
assignment string // ace's assignment value
|
||||
revision time.Time // when the bus wrote it
|
||||
at time.Time // when ace wrote its report
|
||||
counts bool
|
||||
}{
|
||||
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
set := &buildrun.ToolchainSet{}
|
||||
var said []string
|
||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||
key, value := gateReport(t, node, "v26.12.1")
|
||||
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||
}
|
||||
gate.Settled()
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||
t.Fatal("an assignment with no report passed something on")
|
||||
}
|
||||
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||
}
|
||||
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||
t.Fatal("the same assignment written again changed what was passed on")
|
||||
}
|
||||
v, none := agreedOf(set)
|
||||
if c.counts {
|
||||
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||
}
|
||||
return
|
||||
}
|
||||
if v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||
}
|
||||
// ace writes its report again since it was assigned: it counts.
|
||||
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("ace's new report was not passed on")
|
||||
}
|
||||
if n := len(set.Entries("typescript")); n != 4 {
|
||||
t.Fatalf("four builders report; got %d", n)
|
||||
}
|
||||
})
|
||||
key, value := gateReport(t, "g14", "v26.12.1")
|
||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("an assigned node's report was not passed on")
|
||||
}
|
||||
key, value = gateReport(t, "ace", "v26.12.0")
|
||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||
}
|
||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||
t.Fatalf("got %q, %q", v, none)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,8 +10,6 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -75,14 +73,11 @@ func (w *toolchainWatch) ensure(module string) error {
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the module's next report.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): %s's toolchains are not heard until its next report", name, err, module)
|
||||
return nil
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
process.NotWatched(w.logger.Printf, name, err, fmt.Sprintf("%s's toolchains are not heard until its next report", module))
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -22,7 +21,6 @@ type laterBuckets struct {
|
||||
mu sync.Mutex
|
||||
appears map[string]jetstream.KeyValue // buckets that exist now
|
||||
asked map[string]int
|
||||
refuse error // answered for a bucket that does not exist, when set: a bus that refuses it (issue 1945)
|
||||
}
|
||||
|
||||
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
@@ -32,9 +30,6 @@ func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyVa
|
||||
if kv, ok := l.appears[name]; ok {
|
||||
return kv, nil
|
||||
}
|
||||
if l.refuse != nil {
|
||||
return nil, l.refuse
|
||||
}
|
||||
return nil, jetstream.ErrBucketNotFound
|
||||
}
|
||||
|
||||
@@ -105,12 +100,6 @@ func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *test
|
||||
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
|
||||
}
|
||||
var logs bytes.Buffer
|
||||
// 1945: a bus that refuses both buckets, or does not answer, leaves them unwatched, said, and the indexer running.
|
||||
refusing := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}, refuse: errors.New("context deadline exceeded")}
|
||||
if err := newToolchainWatch(ctx, refusing, sink, log.New(&logs, "", 0)).start(); err != nil || !strings.Contains(logs.String(), "cannot be read") {
|
||||
t.Fatalf("a refused toolchains bucket: start %v, log %q; want it said and the start going on", err, logs.String())
|
||||
}
|
||||
logs.Reset()
|
||||
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
|
||||
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053 h1:8Kbp3aJG3jGndDOfmGdMSDgJVEcc1hyu921hVnnlnlQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024231716-571ac521f053/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58 h1:4Damtdqlf6u6nJhvMA99peDy1jmjixtcwFXJ4bWEkLk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232250-9b408db24a58/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95 h1:I8zbgGzRjmgYydPTWo8hmaZW/P/g+qGDUfu3E32/pkU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024232922-99407c37df95/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -71,7 +71,7 @@ func (ix *Indexer) logf(format string, args ...any) {
|
||||
}
|
||||
|
||||
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
|
||||
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
|
||||
// pull request, a closed one takes it out, a merged one marks it merged; a group left with none open is deleted. A change is written
|
||||
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
||||
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
||||
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
@@ -154,12 +154,13 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
|
||||
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
||||
}
|
||||
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
|
||||
return !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
|
||||
// a member that merged is open no more, and stays while another member is (issue 1791)
|
||||
return !p.Merged && !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
|
||||
})
|
||||
switch {
|
||||
case len(kept) == len(c.Pulls):
|
||||
return nil, false, nil
|
||||
case len(kept) == 0:
|
||||
case !anyOpen(kept):
|
||||
ix.logf("%s: no pull request of the group is open, so no candidate", key)
|
||||
return nil, true, nil
|
||||
}
|
||||
@@ -232,7 +233,16 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
|
||||
if at < 0 {
|
||||
return cur
|
||||
}
|
||||
pulls = slices.Delete(pulls, at, at+1)
|
||||
if f.Kind == delivery.ForgeMerged {
|
||||
// A member that merges stays, marked merged, with the head it merged at: the heads hash the group's Approve
|
||||
// covers holds, so the checker starts no new check for the members still open (issue 1791; issue 1921 S9).
|
||||
pulls[at].Merged = true
|
||||
} else {
|
||||
pulls = slices.Delete(pulls, at, at+1)
|
||||
}
|
||||
if !anyOpen(pulls) {
|
||||
return nil // every member merged or closed: the group is done
|
||||
}
|
||||
} else {
|
||||
entry := delivery.CandidatePull{Head: delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}, Modules: []string{}}
|
||||
switch {
|
||||
@@ -276,6 +286,11 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[s
|
||||
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
|
||||
}
|
||||
|
||||
// anyOpen says whether a group's pull requests hold one that has not merged.
|
||||
func anyOpen(pulls []delivery.CandidatePull) bool {
|
||||
return slices.ContainsFunc(pulls, func(p delivery.CandidatePull) bool { return !p.Merged })
|
||||
}
|
||||
|
||||
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
|
||||
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
|
||||
const NoneNoPreview = "no merge preview is computed"
|
||||
|
||||
@@ -206,8 +206,8 @@ func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5
|
||||
for _, how := range []string{"closed", "merged"} {
|
||||
func TestClosedTakesAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5 (a merge: TestAMergedMemberStays...)
|
||||
for _, how := range []string{"closed"} {
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
@@ -387,3 +387,70 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
|
||||
t.Errorf("after an empty open message the candidates hold %v", st.vals)
|
||||
}
|
||||
}
|
||||
|
||||
// 1791 (issue 1921, try speed S9): a member of a group that merges while another is open stays in the candidate, marked
|
||||
// merged, with its head: the heads hash the Approve covers holds, so no new check starts for the members left. An open
|
||||
// message does not drop it, a close of the last open member deletes the group, and the last merge does too.
|
||||
func TestAMergedMemberStaysInItsGroupUntilEveryMemberMerged(t *testing.T) {
|
||||
merged := func(repo string, n int, head string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
|
||||
}
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
for _, f := range []delivery.Forge{pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"),
|
||||
pull("novox/mesh-sdk", 7, "feat/x", h3)} {
|
||||
if err := ix.Apply(ctx, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, _ := read(t, st, "feat/x")
|
||||
if err := ix.Apply(ctx, merged("novox/mesh-sdk", 7, h3)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "feat/x")
|
||||
if !ok || len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
|
||||
t.Fatalf("after two merges the candidate = %+v (kept %v), want the three members and the same heads hash", c, ok)
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if want := p.Repository != "novox/mesh-catalog"; p.Merged != want {
|
||||
t.Errorf("%s#%d merged = %v, want %v", p.Repository, p.Number, p.Merged, want)
|
||||
}
|
||||
}
|
||||
// The open message names only the member still open: the merged ones stay.
|
||||
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _ := read(t, st, "feat/x"); len(c.Pulls) != 3 || c.HeadsHash != before.HeadsHash {
|
||||
t.Errorf("the open message dropped a merged member: %+v", c)
|
||||
}
|
||||
// The last open member merges: the group is done.
|
||||
if err := ix.Apply(ctx, merged("novox/mesh-catalog", 300, h2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := read(t, st, "feat/x"); ok {
|
||||
t.Error("every member merged and the group is still a candidate")
|
||||
}
|
||||
// A close of the last open member deletes the group too.
|
||||
ix, st, _ = newIndexer(t)
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
|
||||
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
|
||||
closed := pull("novox/mesh-catalog", 300, "feat/x", h2)
|
||||
closed.Action = delivery.PullClosed
|
||||
if err := ix.Apply(ctx, closed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := read(t, st, "feat/x"); ok {
|
||||
t.Error("the last open member closed and the group, holding only a merged one, is still a candidate")
|
||||
}
|
||||
// A lone pull request that merges leaves no group.
|
||||
ix, st, _ = newIndexer(t)
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(ctx, merged("novox/mesh-delivery", 12, h1))
|
||||
if _, ok := read(t, st, "feat/x"); ok {
|
||||
t.Error("a lone merged pull request is still a candidate")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,8 +22,7 @@ import (
|
||||
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
|
||||
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
|
||||
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
|
||||
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||
// A value the module already holds is not written again.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
digests, err := ix.keepManifests(ctx, f)
|
||||
if err != nil {
|
||||
|
||||
@@ -85,51 +85,14 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
|
||||
|
||||
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
|
||||
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing); the operator 2026-10-25 (a module's value moves only with the module)
|
||||
const repo, next = "novox/mesh-delivery", "cccccccccccccccccccccccccccccccccccccccc"
|
||||
three := map[string]string{"modules/a": "treeA", "modules/b": "treeB", "modules/c": "treeC"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
then func(ix *Indexer) error // after the first merge of a, b and c at merge, stamped by source revision 7
|
||||
// writes is the values written by then; commit is each module's commit after.
|
||||
writes int
|
||||
commit map[string]string
|
||||
}{
|
||||
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
|
||||
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge},
|
||||
then: func(ix *Indexer) error {
|
||||
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||
then: func(ix *Indexer) error {
|
||||
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t)
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.ApplySource(ctx, 7, sourceOf(repo, merge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writes := mods.writes
|
||||
if err := tc.then(ix); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := mods.writes - writes; got != tc.writes {
|
||||
t.Errorf("%d values written, want %d", got, tc.writes)
|
||||
}
|
||||
for module, want := range tc.commit {
|
||||
if m, ok := moduleOf(t, mods, module); !ok || m.Commit != want {
|
||||
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
|
||||
_ = ix.Apply(context.Background(), m)
|
||||
rev, writes := mods.revs["a"], mods.writes
|
||||
_ = ix.Apply(context.Background(), m)
|
||||
if mods.revs["a"] != rev || mods.writes != writes {
|
||||
t.Errorf("the same merge wrote again (%d writes, was %d)", mods.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -112,12 +112,12 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
|
||||
// previewOf is the preview of a head merged onto the trunk head the source names now: found, or stale when only a preview
|
||||
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
|
||||
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv delivery.Preview, found, stale bool) {
|
||||
func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, found, stale bool) {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
trunk := heads[repository]
|
||||
trunk := ix.source.heads[repository]
|
||||
if trunk == "" {
|
||||
return delivery.Preview{}, false, false
|
||||
}
|
||||
@@ -137,23 +137,14 @@ func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (
|
||||
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
|
||||
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
|
||||
// no preview keeps what it had.
|
||||
//
|
||||
// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written
|
||||
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
|
||||
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
|
||||
// fingerprint first appeared at.
|
||||
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
|
||||
ix.mu.Lock()
|
||||
rev, heads := ix.source.rev, ix.source.heads
|
||||
ix.mu.Unlock()
|
||||
changed, read := false, false
|
||||
changed := false
|
||||
for i := range c.Pulls {
|
||||
p := &c.Pulls[i]
|
||||
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
|
||||
pv, found, stale := ix.previewOf(p.Repository, p.Head.Head)
|
||||
if !found && !stale {
|
||||
continue
|
||||
}
|
||||
read = true
|
||||
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
|
||||
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
|
||||
dirs := map[string]string{}
|
||||
@@ -207,9 +198,6 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
|
||||
if changed {
|
||||
c.FPSet = delivery.FingerprintSet(c.Pulls)
|
||||
}
|
||||
if read && rev != 0 && c.Base != rev && (changed || c.Base == 0) {
|
||||
c.Base, changed = rev, true
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -89,16 +87,9 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
|
||||
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
|
||||
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
|
||||
}
|
||||
c := candidateOf(t, cands)
|
||||
if c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
|
||||
if c := candidateOf(t, cands); c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
|
||||
t.Errorf("fpset = %q", c.FPSet)
|
||||
}
|
||||
// The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk):
|
||||
// the builder merges the preview onto that head, never onto the module value's commit.
|
||||
if c.Base != 5 {
|
||||
t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base)
|
||||
}
|
||||
written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c})
|
||||
// Another tree gives another fingerprint.
|
||||
pv := avahiPreview(t, trunkNow, h1)
|
||||
m := pv.Modules["modules/avahi"]
|
||||
@@ -200,38 +191,3 @@ func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *t
|
||||
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
|
||||
}
|
||||
}
|
||||
|
||||
// writtenCandidate is what the builder's tests read of the indexer (mesh-delivery-builder internal/build): a candidate as the
|
||||
// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested
|
||||
// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again.
|
||||
type writtenCandidate struct {
|
||||
Source sourceRev `json:"source"`
|
||||
Candidate delivery.Candidate `json:"candidate"`
|
||||
}
|
||||
|
||||
type sourceRev struct {
|
||||
Rev uint64 `json:"rev"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
}
|
||||
|
||||
func written(t *testing.T, w writtenCandidate) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(w, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = append(data, '\n')
|
||||
const file = "testdata/candidate-written.json"
|
||||
if os.Getenv("MESH_WRITE_TESTDATA") == "1" {
|
||||
if err := os.WriteFile(file, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kept, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(kept) != string(data) {
|
||||
t.Errorf("%s is not what the indexer writes now (MESH_WRITE_TESTDATA=1 writes it again):\n%s", file, data)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,8 +32,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 {
|
||||
}
|
||||
|
||||
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
|
||||
// repository the source names, each module built from it whose fingerprint was computed at the head the source names, and
|
||||
// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped
|
||||
// repository the source names, each module built from it whose fingerprint was computed at the head the source names is
|
||||
// written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped
|
||||
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
|
||||
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
|
||||
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
|
||||
@@ -71,9 +71,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
return nil, false, nil
|
||||
}
|
||||
// Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another
|
||||
// repository) leaves the value as it is, so no revision of the source rewrites a module that did not change.
|
||||
if cur.Repository != repository || cur.Commit != head || cur.Src != 0 {
|
||||
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur.Src = rev
|
||||
|
||||
@@ -38,15 +38,15 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
|
||||
if c, ok := moduleOf(t, mods, "c"); ok {
|
||||
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
|
||||
}
|
||||
// The same revision again changes nothing, and so does a later one: Src is the first revision naming the commit.
|
||||
// The same revision again changes nothing; a later revision moves Src.
|
||||
if err := ix.ApplySource(ctx, 7, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.ApplySource(ctx, 9, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 7 {
|
||||
t.Errorf("module a Src = %d, want 7", m.Src)
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"source": {
|
||||
"rev": 5,
|
||||
"heads": {
|
||||
"novox/mesh-catalog": "5555555555555555555555555555555555555555"
|
||||
}
|
||||
},
|
||||
"candidate": {
|
||||
"v": 1,
|
||||
"group": "feat/x",
|
||||
"base": 5,
|
||||
"heads": "dfe9fb8f79e8",
|
||||
"pulls": [
|
||||
{
|
||||
"repository": "novox/mesh-catalog",
|
||||
"number": 301,
|
||||
"head": "0123456789abcdef0123456789abcdef01234567",
|
||||
"modules": [
|
||||
"avahi"
|
||||
],
|
||||
"fps": {
|
||||
"avahi": {
|
||||
"fp2": "src2:e8d595aea1b13f1f5578ef1f245dab0d28a351c28079ecdcd2276c63a932d7be",
|
||||
"manifest": "8a3856b6c0566d76f2556fc4f46c5251dc7935517bb6bd15809ded7e5546a0d8"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"fpset": "01c840ec276b"
|
||||
}
|
||||
}
|
||||
@@ -112,9 +112,6 @@ type known struct {
|
||||
languages map[string]string
|
||||
bases []string
|
||||
contexts bool
|
||||
// held is the fingerprint the bucket holds for the module and the toolchains it was computed with (for a merge, the
|
||||
// value before it): what stillComputed keeps while the builders agree on no toolchain.
|
||||
held delivery.Module
|
||||
}
|
||||
|
||||
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
|
||||
@@ -162,23 +159,19 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
stale := map[string]bool{} // values an SDK that could not read their manifest wrote, judged again (issue 1923)
|
||||
all := map[string]*known{}
|
||||
for _, key := range keys {
|
||||
cur, merged := overlay[key]
|
||||
var before delivery.Module
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
if found {
|
||||
if before, err = delivery.DecodeModule(value); err != nil && !merged {
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
continue
|
||||
cur, ok := overlay[key]
|
||||
if !ok {
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
if !merged {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
cur = before
|
||||
if cur, err = delivery.DecodeModule(value); err != nil {
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
held[key] = cur
|
||||
if cur.Manifest == "" || cur.Inputs2 == nil {
|
||||
@@ -193,7 +186,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
cur, stale[key] = again, true
|
||||
held[key] = cur
|
||||
}
|
||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
|
||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
|
||||
}
|
||||
tools := ix.agreed()
|
||||
type result struct {
|
||||
@@ -258,9 +251,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
r.none = strings.Join(missing, "; ")
|
||||
case noToolchain != "":
|
||||
r.none = noToolchain
|
||||
if fp, ok := stillComputed(module, kept, k, toolchains); ok {
|
||||
r = result{fp: fp, tcs: k.held.Toolchains}
|
||||
}
|
||||
default:
|
||||
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
|
||||
if r.fp != "" {
|
||||
@@ -319,12 +309,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
||||
|
||||
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
||||
// the ones that were computed, the others are written as the merge gave them.
|
||||
//
|
||||
// **A module the merge did not change is not written** (the operator, 2026-10-25: "only if a module has got a new version in
|
||||
// commit x, should a new module version get created linked to commit x"). A value names the commit its fingerprint first
|
||||
// appeared at: when the merge gives a module the value it holds but for the commit (and the source revision stamped on it),
|
||||
// the held value stands, its commit unchanged, so a catalogue merge touching one module writes one key, not one per module of
|
||||
// the repository, and the controller's take-in hears only that one. The repository's trunk head is the gate's source.main.
|
||||
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
|
||||
for _, key := range slices.Sorted(maps.Keys(overlay)) {
|
||||
next := overlay[key]
|
||||
@@ -333,7 +317,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
||||
}
|
||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && sameButWhere(cur, next) {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
@@ -347,13 +331,6 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
||||
return nil
|
||||
}
|
||||
|
||||
// sameButWhere says the held value cur is next but for where it was computed (Commit and Src): the module is unchanged.
|
||||
func sameButWhere(cur, next delivery.Module) bool {
|
||||
next = withV(next)
|
||||
next.Commit, next.Src = cur.Commit, cur.Src
|
||||
return reflect.DeepEqual(cur, next)
|
||||
}
|
||||
|
||||
// manifestRaw is manifests.<digest>: the manifest as stored and parsed by the running SDK.
|
||||
func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, catalogue.Manifest, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
@@ -404,32 +381,6 @@ func hasContext(m catalogue.Manifest) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// stillComputed is the fingerprint a module already holds, kept while a language it is built with has no agreed toolchain
|
||||
// (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's
|
||||
// fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were
|
||||
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
|
||||
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
|
||||
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
|
||||
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2) (string, bool) {
|
||||
if k.held.FP2 == "" {
|
||||
return "", false
|
||||
}
|
||||
toolchains := map[string]delivery.Toolchain2{}
|
||||
for artifact, language := range k.languages {
|
||||
if tc, ok := agreed[artifact]; ok {
|
||||
toolchains[artifact] = tc
|
||||
continue
|
||||
}
|
||||
used, ok := k.held.Toolchains[language]
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
|
||||
}
|
||||
fp, none := delivery.SourceFingerprint2(kept.With(module, toolchains))
|
||||
return fp, none == "" && fp == k.held.FP2
|
||||
}
|
||||
|
||||
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
|
||||
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
|
||||
if len(by) == 0 {
|
||||
|
||||
@@ -123,16 +123,9 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
|
||||
}
|
||||
|
||||
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
|
||||
// Both reports are read before the module's first fingerprint is computed (a fingerprint computed before they
|
||||
// disagree is kept: TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith).
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
for node, version := range map[string]string{"novox": "go1.27.1", "ace": "go1.26.0"} {
|
||||
key, _ := delivery.ToolchainKey(node, "go")
|
||||
ix.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", version, "1"), false)
|
||||
}
|
||||
if err := ix.Recompute(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
|
||||
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
|
||||
@@ -267,13 +260,11 @@ func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (revie
|
||||
w := mods.writes
|
||||
// the same catalogue merged again at a later commit: nothing moved
|
||||
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
f.Merge = otherMerge
|
||||
if err := ix.Apply(ctx, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// and the commit stays the one the fingerprint first appeared at (the operator, 2026-10-25)
|
||||
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || after.Commit != before.Commit || mods.writes != w {
|
||||
t.Errorf("fp2 = %q at %.8s (was %q at %.8s), writes %d more", after.FP2, after.Commit, before.FP2, before.Commit, mods.writes-w)
|
||||
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || mods.writes != w {
|
||||
t.Errorf("fp2 = %q (was %q), writes %d more", after.FP2, before.FP2, mods.writes-w)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,24 +288,9 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
|
||||
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
|
||||
}
|
||||
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
|
||||
// stale report blanked every module and the controller's take-in refused modules already built); only a new
|
||||
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
|
||||
// Nodes that disagree leave no fingerprint and no toolchains.
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != n.FP2 || !reflect.DeepEqual(d.Toolchains, n.Toolchains) || d.None2 != "" {
|
||||
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v none2 %q, want the fingerprint kept (%q) with %+v", d.FP2, d.Toolchains, d.None2, n.FP2, n.Toolchains)
|
||||
}
|
||||
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
f.Trees["modules/avahi"] = "tree-avahi-moved"
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil || !strings.HasPrefix(d.None2, "builders disagree") {
|
||||
t.Errorf("a new tree while nodes disagree: fp2 %q toolchains %+v none2 %q, want no fingerprint and the disagreement", d.FP2, d.Toolchains, d.None2)
|
||||
}
|
||||
// They agree again: the new tree gets its fingerprint.
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.2")
|
||||
if a, _ := moduleOf(t, mods, "avahi"); a.FP2 == "" || a.FP2 == n.FP2 {
|
||||
t.Errorf("agreed again: fp2 %q (was %q), want a new fingerprint", a.FP2, n.FP2)
|
||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil {
|
||||
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v, want neither", d.FP2, d.Toolchains)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The assigner's asks reach the operator (issue 1906). The assigner writes its asks (the hold of a fingerprint, the restore
|
||||
// of a data module, the backup before an apply and a move of a module of policy record) in its bucket asks; until now the
|
||||
// teller only filled them from an answer given in the view, so an ask nobody opened the view for reached nobody: on
|
||||
// 2026-10-25 seven modules stayed held on every machine and the operator was never asked. Now the teller raises each on the
|
||||
// operator channel as it raises the gate's merge asks: as the asker mesh-delivery-teller, once per router id, through the same
|
||||
// records, the same wait for room and the same warrants. The operator's answer, from the phone or from the view, is written
|
||||
// under the ask's own key in the bucket answers, which the assigner reads (mesh-delivery-assigner pass.go answeredWith); none
|
||||
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
|
||||
|
||||
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
|
||||
// asks.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
|
||||
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
|
||||
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
|
||||
const mostAssignerOpen = asks.MostOpen - 1
|
||||
|
||||
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
|
||||
const assignerOpenRecord = "assigner.open"
|
||||
|
||||
// assignerOpen is the router ids of the assigner's asks open now: those asked whose ask is neither answered nor ended. An id
|
||||
// that ended is dropped from the record as it is read.
|
||||
func (t *Teller) assignerOpen() ([]string, *stdio.StateEntry) {
|
||||
e, err := t.Store.Get(assignerOpenRecord)
|
||||
if err != nil || e == nil {
|
||||
return nil, e
|
||||
}
|
||||
var ids []string
|
||||
if err := json.Unmarshal(e.Value, &ids); err != nil {
|
||||
t.logf("the record %s cannot be read: %v", assignerOpenRecord, err)
|
||||
return nil, e
|
||||
}
|
||||
return slices.DeleteFunc(ids, func(id string) bool {
|
||||
k, _, err := t.keptOf(id)
|
||||
return err == nil && !open(k)
|
||||
}), e
|
||||
}
|
||||
|
||||
// assignerRoom says whether another of the assigner's asks may be asked now.
|
||||
func (t *Teller) assignerRoom() bool {
|
||||
ids, _ := t.assignerOpen()
|
||||
return len(ids) < mostAssignerOpen
|
||||
}
|
||||
|
||||
// assignerAsked keeps that the assigner's ask with this router id is asked.
|
||||
func (t *Teller) assignerAsked(id string) {
|
||||
ids, e := t.assignerOpen()
|
||||
if !slices.Contains(ids, id) {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
var err error
|
||||
if e == nil {
|
||||
_, err = t.Store.Create(assignerOpenRecord, ids)
|
||||
} else {
|
||||
_, err = t.Store.Update(assignerOpenRecord, ids, e.Revision)
|
||||
}
|
||||
if err != nil {
|
||||
t.logf("the record %s could not be kept: %v", assignerOpenRecord, err)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForAssignerRoom puts an assigner ask in the wait for room, once.
|
||||
func (t *Teller) waitForAssignerRoom(key, id string) {
|
||||
ws, rec, err := t.waiting()
|
||||
if err != nil {
|
||||
t.logf("%v", err)
|
||||
return
|
||||
}
|
||||
if slices.ContainsFunc(ws, func(x waitingAsk) bool { return x.ID == id }) {
|
||||
return
|
||||
}
|
||||
if t.keepWaiting(append(ws, waitingAsk{Key: key, ID: id}), rec) == nil {
|
||||
t.logf("%s: the assigner's ask %s waits: %d of the assigner's asks are open, the most the teller keeps open so a merge ask always has room", key, id, mostAssignerOpen)
|
||||
}
|
||||
}
|
||||
|
||||
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
|
||||
func isAssignerKey(key string) bool {
|
||||
k, err := delivery.ParseAnyAskKey(key)
|
||||
return err == nil && k.Kind != delivery.AskKindMerge
|
||||
}
|
||||
|
||||
// assignerAsk reads the assigner's ask at key: found false when the key holds none.
|
||||
func (t *Teller) assignerAsk(key string) (delivery.AssignerAsk, bool, error) {
|
||||
if t.Assigner == nil {
|
||||
return delivery.AssignerAsk{}, false, fmt.Errorf("the teller reads no assigner asks")
|
||||
}
|
||||
raw, found, err := t.Assigner.Get(key)
|
||||
if err != nil || !found {
|
||||
return delivery.AssignerAsk{}, false, err
|
||||
}
|
||||
a, err := delivery.DecodeAssignerAsk(key, raw)
|
||||
if err != nil {
|
||||
return delivery.AssignerAsk{}, false, fmt.Errorf("the assigner's key %s holds no ask: %w", key, err)
|
||||
}
|
||||
return a, true, nil
|
||||
}
|
||||
|
||||
// raiseAssignerAsk asks the operator the assigner's ask at key, once per router id: an ask answered already (from the view or
|
||||
// the phone) is not asked; an earlier asking still open is taken back first; an asking already made is only caught up with
|
||||
// at the router, and one that ended is not asked again, for the router drops a second ask under an id it keeps.
|
||||
func (t *Teller) raiseAssignerAsk(key string) {
|
||||
a, found, err := t.assignerAsk(key)
|
||||
if err != nil {
|
||||
t.logf("%s: the ask is not raised: %v", key, err)
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
id := a.Ask.ID
|
||||
if raw, _, has, err := t.Answers.Get(delivery.AnswerKey(key)); err != nil {
|
||||
t.logf("%s: the answers could not be read, so the ask is not raised: %v", key, err)
|
||||
return
|
||||
} else if has && answeredAsking(raw) == id {
|
||||
return
|
||||
}
|
||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
return
|
||||
}
|
||||
if grec != nil && prior.ID != id {
|
||||
t.cancelOpen(key, prior.ID) // a new asking of the same key: the earlier one is taken back
|
||||
}
|
||||
k, _, err := t.keptOf(id)
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
return
|
||||
}
|
||||
if k != nil {
|
||||
if grec == nil || prior.ID != id {
|
||||
_ = t.note(id, key, 0, grec)
|
||||
}
|
||||
if open(k) {
|
||||
t.catchUp(id)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := t.note(id, key, 0, grec); err != nil {
|
||||
t.logf("%s: the record of the ask %s could not be kept, so it is not raised: %v", key, id, err)
|
||||
return
|
||||
}
|
||||
if !t.assignerRoom() {
|
||||
t.waitForAssignerRoom(key, id)
|
||||
return
|
||||
}
|
||||
if err := t.Asker.Ask(a.Ask); err != nil {
|
||||
t.logf("%s: the ask %s was not asked: %v", key, id, err)
|
||||
if kept, _, _ := t.keptOf(id); open(kept) {
|
||||
_ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it
|
||||
}
|
||||
return
|
||||
}
|
||||
t.assignerAsked(id)
|
||||
t.logf("%s: the assigner's ask %s is asked of the operator (%s)", key, id, a.Ask.Headline)
|
||||
}
|
||||
|
||||
// OnAssignerAskGone handles a key the assigner took away from its asks (a hold it lifted by itself): the open router ask of
|
||||
// its latest asking is taken back, so the operator is not asked about what no longer holds.
|
||||
func (t *Teller) OnAssignerAskGone(key string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if !isAssignerKey(key) {
|
||||
t.logf("the assigner's key %s was taken away and is no ask key of the assigner, and is left", key)
|
||||
return
|
||||
}
|
||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||
if err != nil || grec == nil {
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.cancelOpen(key, prior.ID)
|
||||
}
|
||||
|
||||
// answerAssigner is the act of a warrant for one of the assigner's asks: the ask must still stand at its key under the asking
|
||||
// and the words the operator answered, and then the answer is written under the ask's key, once. Its options bind no act:
|
||||
// the assigner reads the option chosen (asker.Client.Take has checked the option, its level and the ask's digest).
|
||||
func (t *Teller) answerAssigner(key string, o asks.Option, w asks.Warrant) error {
|
||||
a, found, err := t.assignerAsk(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
t.logf("%s: the ask %s was answered and the assigner has taken it away: the answer changes nothing", key, w.Ask)
|
||||
return nil
|
||||
}
|
||||
if a.Ask.ID != w.Ask {
|
||||
t.logf("%s: the ask %s was answered, and the assigner has asked again under %s: an answer to an earlier asking changes nothing", key, w.Ask, a.Ask.ID)
|
||||
return nil
|
||||
}
|
||||
if a.Ask.Digest() != w.AskDigest {
|
||||
return fmt.Errorf("the assigner's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)", key, a.Ask.Digest(), w.AskDigest)
|
||||
}
|
||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.writeAnswer(key, w.Ask, body, w.Says())
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
|
||||
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
|
||||
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
b.t.OnAssignerAsk(key)
|
||||
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
|
||||
}
|
||||
var got asks.Ask
|
||||
if err := json.Unmarshal(asked[0].body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.ID != ask.ID || got.Digest() != ask.Digest() {
|
||||
t.Errorf("the router's ask %+v differs from the assigner's", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: without the view's bucket the assigner's asks are still raised.
|
||||
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner := memGate{}
|
||||
b.t.Assigner = assigner
|
||||
key, value, _ := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
|
||||
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
|
||||
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
|
||||
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
w := ackFor(ask, "release")
|
||||
b.deliver(t, w)
|
||||
raw, ok := b.answers.written[delivery.AnswerKey(key)]
|
||||
if !ok {
|
||||
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
|
||||
}
|
||||
if a, err := delivery.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
|
||||
t.Errorf("answer = %+v, %v", a, err)
|
||||
}
|
||||
b.deliver(t, w)
|
||||
if b.answers.writes != 1 || len(b.conds) != 0 {
|
||||
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
|
||||
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
delete(assigner, key)
|
||||
b.deliver(t, ackFor(ask, "release"))
|
||||
if b.answers.writes != 0 {
|
||||
t.Errorf("an answer to a withdrawn ask was written")
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
|
||||
// itself) is taken back too; an ask answered before it is heard is never raised.
|
||||
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
|
||||
cancelled := func(b *bed, id string) bool {
|
||||
for _, p := range b.askedOf(asks.CancelSubject(delivery.TellerModule)) {
|
||||
if string(p.body) == `{"id":"`+id+`"}` {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
b := newBed(t)
|
||||
assigner, view := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
view[key+".view"] = viewAnswer(t, "release", "")
|
||||
b.t.OnViewAnswer(key + ".view")
|
||||
if !cancelled(b, ask.ID) {
|
||||
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
|
||||
}
|
||||
|
||||
b = newBed(t)
|
||||
assigner, _ = withView(b)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
delete(assigner, key)
|
||||
b.t.OnAssignerAskGone(key)
|
||||
if !cancelled(b, ask.ID) {
|
||||
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
|
||||
}
|
||||
|
||||
b = newBed(t)
|
||||
assigner, view = withView(b)
|
||||
assigner[key] = value
|
||||
view[key+".view"] = viewAnswer(t, "keep", "")
|
||||
b.t.OnAssignerAsk(key)
|
||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 0 {
|
||||
t.Errorf("an ask answered in the view was raised %d times", n)
|
||||
}
|
||||
}
|
||||
|
||||
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
|
||||
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
|
||||
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
_, first := b.askOf(t, 1, 1)
|
||||
b.askOf(t, 2, 2)
|
||||
b.askOf(t, 3, 3)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
b.refuse(t, ask.ID, roomRefusal)
|
||||
b.deliver(t, warrantFor(first.Ask, "approve"))
|
||||
got := b.askedIDs()
|
||||
if len(got) != 5 || got[4] != ask.ID {
|
||||
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
|
||||
}
|
||||
|
||||
b = newBed(t)
|
||||
assigner, _ = withView(b)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
b.refuse(t, ask.ID, "the ask has no headline")
|
||||
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
|
||||
t.Errorf("conditions = %+v", b.conds)
|
||||
}
|
||||
}
|
||||
|
||||
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
|
||||
func ackFor(a asks.Ask, option string) asks.Warrant {
|
||||
w := warrantFor(a, option)
|
||||
w.Level, w.Label = asks.Acknowledge, option
|
||||
return w
|
||||
}
|
||||
|
||||
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
|
||||
func holdAskOf(t *testing.T, n int) (string, []byte, asks.Ask) {
|
||||
t.Helper()
|
||||
key, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, ask := holdAsk(t)
|
||||
ask.ID = delivery.AskID(key)
|
||||
value, err := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value, ask
|
||||
}
|
||||
|
||||
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
|
||||
// ask is still asked, and each hold answered lets the next waiting one be asked.
|
||||
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
|
||||
b := newBed(t)
|
||||
assigner, _ := withView(b)
|
||||
var held []asks.Ask
|
||||
for n := 0; n < 7; n++ {
|
||||
key, value, ask := holdAskOf(t, n)
|
||||
assigner[key] = value
|
||||
b.t.OnAssignerAsk(key)
|
||||
held = append(held, ask)
|
||||
}
|
||||
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
|
||||
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
|
||||
}
|
||||
_, merge := b.askOf(t, 1, 1)
|
||||
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
|
||||
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
|
||||
}
|
||||
if len(b.dropped) != 0 || len(b.conds) != 0 {
|
||||
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
|
||||
}
|
||||
b.deliver(t, ackFor(held[0], "release"))
|
||||
got := b.askedIDs()
|
||||
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
|
||||
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
|
||||
}
|
||||
// The merge ask answered frees its own place, and no hold takes it.
|
||||
b.deliver(t, warrantFor(merge.Ask, "approve"))
|
||||
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
|
||||
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
|
||||
}
|
||||
}
|
||||
@@ -524,9 +524,8 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
||||
t.Fatalf("answer %+v err %v", a, err)
|
||||
}
|
||||
// The view's answer written again with the same option changes nothing under the ask (another option, later, is the
|
||||
// operator changing their mind: TestTellerTakesAChangedHoldAnswerFromTheView, issue 1906 note 23).
|
||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "")); err != nil {
|
||||
// The view's answer written again changes nothing under the ask.
|
||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "keep", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
@@ -546,148 +545,3 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
||||
t.Error("the merge's ask was answered from a view key")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer starts the program against the bus: the assigner's hold ask, put in
|
||||
// its bucket, is on the operator channel's queue as the teller's ask; the router's warrant for "release" is written under the
|
||||
// ask's key, where the assigner reads it; a second ask the assigner takes away (a hold it lifted) is cancelled at the router
|
||||
// (1906/H1, H2 on the real bus).
|
||||
func TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer(t *testing.T) {
|
||||
now = time.Now().UTC()
|
||||
b := startFull(t)
|
||||
assigner := b.as(t, "n1.mesh-delivery-assigner")
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
logs := &syncBuffer{}
|
||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||
|
||||
key, value, ask := holdAsk(t)
|
||||
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond) // the program hears warrants and watches
|
||||
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raw *jetstream.RawStreamMsg
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw == nil {
|
||||
t.Fatalf("the assigner's ask is not on the seat's queue: %v\n%s", err, logs.String())
|
||||
}
|
||||
var asked asks.Ask
|
||||
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.ID != ask.ID || asked.Digest() != ask.Digest() {
|
||||
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
|
||||
}
|
||||
w, _ := json.Marshal(ackFor(ask, "release"))
|
||||
if err := b.ctl.Publish(seatDecided, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var e jetstream.KeyValueEntry
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if e == nil {
|
||||
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
|
||||
}
|
||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
||||
t.Fatalf("answer %+v err %v", a, err)
|
||||
}
|
||||
|
||||
key2, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask2 := ask
|
||||
ask2.ID = delivery.AskID(key2)
|
||||
value2, err := delivery.EncodeAssignerAsk(key2, delivery.AssignerAsk{Ask: ask2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := askKV.Put(b.ctx, key2, value2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if strings.Contains(logs.String(), ask2.ID+" is asked") {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := askKV.Delete(b.ctx, key2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
|
||||
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestTellerTakesAChangedHoldAnswerFromTheView starts the program against the bus: a hold ask answered "keep" earlier, and the
|
||||
// view's answer "release" written now, replace the answer of that asking with "release" (1906/N23 on the real bus).
|
||||
func TestTellerTakesAChangedHoldAnswerFromTheView(t *testing.T) {
|
||||
now = time.Now().UTC()
|
||||
b := startFull(t)
|
||||
assigner, view := b.as(t, "n1.mesh-delivery-assigner"), b.as(t, "n1.mesh-delivery-view")
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, value, ask := holdAsk(t)
|
||||
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator chose keep"})
|
||||
if _, err := answers.Put(b.ctx, delivery.AnswerKey(key), kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
logs := &syncBuffer{}
|
||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||
viewKV, err := view.KeyValue(b.ctx, viewAnswers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond) // the program is watching
|
||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "it was a flake")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
||||
if a, err := delivery.DecodeAnswer(e.Value()); err == nil && a.Option == "release" && answeredAsking(e.Value()) == ask.ID {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("the kept hold's answer was not changed to release\n%s", logs.String())
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -108,15 +107,3 @@ func (g kvGate) Get(key string) ([]byte, bool, error) {
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}
|
||||
|
||||
// GetAt is Get with the time the value was written.
|
||||
func (g kvGate) GetAt(key string) ([]byte, time.Time, bool, error) {
|
||||
e, err := g.kv.Get(g.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, time.Time{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, time.Time{}, false, err
|
||||
}
|
||||
return e.Value(), e.Created(), true, nil
|
||||
}
|
||||
|
||||
@@ -22,8 +22,6 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
@@ -121,14 +119,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
// Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant).
|
||||
return process.EssentialErr("teller", delivery.AsksBucket, "tell any of the gate's asks", err)
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err)
|
||||
}
|
||||
// The answers it writes and what it asked: its own work, so each stops the start, naming its grant.
|
||||
var kvs [2]jetstream.KeyValue
|
||||
for i, name := range []string{delivery.AnswersBucket, askedBucket} {
|
||||
if kvs[i], err = process.EssentialBucket(ctx, js, "teller", name, "write an answer or remember what it asked"); err != nil {
|
||||
return err
|
||||
if kvs[i], err = js.KeyValue(ctx, name); err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
state := kvStore{ctx: ctx, kv: kvs[1]}
|
||||
@@ -144,21 +140,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not
|
||||
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
||||
failed := make(chan error, 2)
|
||||
assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf)
|
||||
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
|
||||
if viewKV != nil {
|
||||
t.View = kvGate{ctx: ctx, kv: viewKV}
|
||||
}
|
||||
if assignerKV != nil {
|
||||
t.Assigner = kvGate{ctx: ctx, kv: assignerKV}
|
||||
go func() {
|
||||
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil)
|
||||
}()
|
||||
}
|
||||
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
||||
if assignerKV != nil && viewKV != nil {
|
||||
t.Assigner, t.View = kvGate{ctx: ctx, kv: assignerKV}, kvGate{ctx: ctx, kv: viewKV}
|
||||
go func() {
|
||||
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
|
||||
}()
|
||||
go func() {
|
||||
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, func(string) {}, nil)
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
|
||||
@@ -176,8 +166,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
|
||||
// teller is what an optional bucket of the teller leaves out.
|
||||
const teller = "the asks and answers it holds are not carried until the teller is started again with it"
|
||||
// optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not.
|
||||
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
logger.Printf("the bucket %s cannot be read (%v): the view's answers fill no ask until the teller is started again with it", name, err)
|
||||
return nil
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
|
||||
// raised (manifest state entry asked, novox/hq ADR 0437 §4).
|
||||
|
||||
@@ -340,9 +340,6 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
||||
if srec == nil {
|
||||
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
|
||||
}
|
||||
if isAssignerKey(s.Key) {
|
||||
return t.answerAssigner(s.Key, o, w)
|
||||
}
|
||||
raw, found, err := t.Gate.Get(s.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
|
||||
@@ -370,31 +367,26 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return t.writeAnswer(s.Key, w.Ask, body, w.Says())
|
||||
}
|
||||
|
||||
// writeAnswer writes the answer to the asking askID under the ask's key by compare-and-set: created when the key has no answer;
|
||||
// updated at the revision read when it holds an earlier asking's answer (the reader ignores that one); left when it holds this
|
||||
// asking's.
|
||||
func (t *Teller) writeAnswer(key, askID string, body []byte, words string) error {
|
||||
akey := delivery.AnswerKey(key)
|
||||
// Compare-and-set: created when the key has no answer; updated at the revision read when it holds an earlier asking's
|
||||
// answer (the gate ignores that one); left when it holds this asking's.
|
||||
akey := delivery.AnswerKey(s.Key)
|
||||
cur, crev, found, err := t.Answers.Get(akey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the answer under %s could not be read: %w", key, err)
|
||||
return fmt.Errorf("the answer under %s could not be read: %w", s.Key, err)
|
||||
}
|
||||
if found && answeredAsking(cur) == askID {
|
||||
t.logf("%s: an answer is there already, and is not written again", key)
|
||||
if found && answeredAsking(cur) == w.Ask {
|
||||
t.logf("%s: an answer is there already, and is not written again", s.Key)
|
||||
return nil
|
||||
}
|
||||
written, err := t.Answers.Write(akey, body, crev, found)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the answer under %s could not be written: %w", key, err)
|
||||
return fmt.Errorf("the answer under %s could not be written: %w", s.Key, err)
|
||||
}
|
||||
if !written {
|
||||
t.logf("%s: another writer answered first, and the answer is not written again", key)
|
||||
t.logf("%s: another writer answered first, and the answer is not written again", s.Key)
|
||||
return nil
|
||||
}
|
||||
t.logf("%s: %s", key, words)
|
||||
t.logf("%s: %s", s.Key, w.Says())
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -431,10 +423,10 @@ func (t *Teller) saidOnce(h asker.Handled, w asks.Warrant, err error) {
|
||||
t.clear(w.Ask)
|
||||
case h == asker.Refused || (h == asker.Acted && err != nil):
|
||||
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
|
||||
t.raise(w.Ask, t.kindOf(w.Ask).answerHeadline, "its answer did not hold here: "+oneLine(err))
|
||||
t.raise(w.Ask, "A merge Approve did not hold", "its answer did not hold here: "+oneLine(err))
|
||||
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
|
||||
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
|
||||
t.raise(w.Ask, t.kindOf(w.Ask).refusedHeadline, "the router refused it: "+w.Words)
|
||||
t.raise(w.Ask, "A merge ask was refused", "the router refused it: "+w.Words)
|
||||
case err != nil:
|
||||
t.logf("the warrant of %s: %v", w.Ask, err)
|
||||
}
|
||||
@@ -454,30 +446,6 @@ func oneLine(err error) string {
|
||||
// conditionKey is the key of the condition about an ask.
|
||||
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
|
||||
|
||||
// askWords are the words the teller's conditions use for one kind of ask: the gate's merge ask, or one of the assigner's.
|
||||
type askWords struct {
|
||||
scope, noun, needs, nothing, resolved string
|
||||
answerHeadline, refusedHeadline, unaskedHeadline string
|
||||
}
|
||||
|
||||
var (
|
||||
mergeWords = askWords{scope: "merge", noun: "The merge ask ", needs: "to ask for the Approve again", nothing: "Nothing was merged.",
|
||||
resolved: "A good answer to the merge ask was written.", answerHeadline: "A merge Approve did not hold",
|
||||
refusedHeadline: "A merge ask was refused", unaskedHeadline: "A merge ask could not be asked"}
|
||||
assignerWords = askWords{scope: "delivery", noun: "The delivery ask ", needs: "to answer it in the delivery view",
|
||||
nothing: "Nothing was changed on a machine.", resolved: "A good answer to the delivery ask was written.",
|
||||
answerHeadline: "An answer to a delivery ask did not hold", refusedHeadline: "A delivery ask was refused",
|
||||
unaskedHeadline: "A delivery ask could not be asked"}
|
||||
)
|
||||
|
||||
// kindOf is the words for the ask with this router id, by the key it was asked for.
|
||||
func (t *Teller) kindOf(askID string) askWords {
|
||||
if isAssignerKey(t.condID(askID)) {
|
||||
return assignerWords
|
||||
}
|
||||
return mergeWords
|
||||
}
|
||||
|
||||
func (t *Teller) raise(askID, headline, why string) {
|
||||
if t.Say == nil {
|
||||
return
|
||||
@@ -489,12 +457,11 @@ func (t *Teller) raise(askID, headline, why string) {
|
||||
}
|
||||
t.logf("the condition of %s could not be kept: %v", id, err)
|
||||
}
|
||||
k := t.kindOf(askID)
|
||||
needs := k.needs
|
||||
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: k.scope, ID: id}, State: asks.StateRaised,
|
||||
Summary: k.noun + id + ": " + why, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". " + k.noun + id + ", " + why + ". " + k.nothing,
|
||||
Resolved: k.resolved}
|
||||
needs := "to ask for the Approve again"
|
||||
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateRaised,
|
||||
Summary: "The merge ask " + id + ": " + why, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". The merge ask " + id + ", " + why + ". Nothing was merged.",
|
||||
Resolved: "A good answer to the merge ask was written."}
|
||||
if err := t.Say(c); err != nil {
|
||||
t.logf("the condition of %s could not be said: %v", id, err)
|
||||
_ = t.Store.Delete(condKey(id))
|
||||
@@ -510,7 +477,7 @@ func (t *Teller) clear(askID string) {
|
||||
if err != nil || e == nil {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: t.kindOf(askID).scope, ID: id}, State: asks.StateCleared}
|
||||
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateCleared}
|
||||
if err := t.Say(c); err != nil {
|
||||
t.logf("the condition of %s could not be cleared: %v", id, err)
|
||||
return
|
||||
@@ -611,7 +578,7 @@ func (t *Teller) roomRefused(w asks.Warrant) {
|
||||
return
|
||||
}
|
||||
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
|
||||
t.raise(w.Ask, t.kindOf(w.Ask).unaskedHeadline, fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
||||
t.raise(w.Ask, "A merge ask could not be asked", fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
||||
return
|
||||
}
|
||||
if t.keepWaiting(ws, rec) != nil {
|
||||
@@ -708,11 +675,6 @@ func (t *Teller) askWaiting(n int) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
assigner := isAssignerKey(x.Key)
|
||||
if assigner && !t.assignerRoom() {
|
||||
i++ // the assigner's asks hold their most open: it waits, and a merge ask after it may be asked
|
||||
continue
|
||||
}
|
||||
a, why := t.stillAsked(x)
|
||||
if why != "" {
|
||||
t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why)
|
||||
@@ -725,9 +687,6 @@ func (t *Teller) askWaiting(n int) {
|
||||
return
|
||||
}
|
||||
ws[i].Asked = true
|
||||
if assigner {
|
||||
t.assignerAsked(x.ID)
|
||||
}
|
||||
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
|
||||
if n--; n <= 0 {
|
||||
return
|
||||
@@ -742,9 +701,6 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
||||
if x.Key == "" {
|
||||
return asks.Ask{}, "the teller holds no gate key for it"
|
||||
}
|
||||
if isAssignerKey(x.Key) {
|
||||
return t.stillAskedByAssigner(x)
|
||||
}
|
||||
raw, found, err := t.Gate.Get(x.Key)
|
||||
if err != nil {
|
||||
return asks.Ask{}, "the gate's ask could not be read: " + err.Error()
|
||||
@@ -768,26 +724,6 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
||||
return ga.Ask, ""
|
||||
}
|
||||
|
||||
// stillAskedByAssigner is stillAsked for one of the assigner's asks.
|
||||
func (t *Teller) stillAskedByAssigner(x waitingAsk) (asks.Ask, string) {
|
||||
a, found, err := t.assignerAsk(x.Key)
|
||||
switch {
|
||||
case err != nil:
|
||||
return asks.Ask{}, err.Error()
|
||||
case !found:
|
||||
return asks.Ask{}, "the assigner has taken it away"
|
||||
case a.Ask.ID != x.ID:
|
||||
return asks.Ask{}, "the assigner asks again under " + a.Ask.ID
|
||||
}
|
||||
if cur, _, has, err := t.Answers.Get(delivery.AnswerKey(x.Key)); err == nil && has && answeredAsking(cur) == x.ID {
|
||||
return asks.Ask{}, "it is answered"
|
||||
}
|
||||
if err := a.Ask.Check(t.now()); err != nil {
|
||||
return asks.Ask{}, "it no longer holds: " + err.Error()
|
||||
}
|
||||
return a.Ask, ""
|
||||
}
|
||||
|
||||
// askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and
|
||||
// publishes it under the bus message id given.
|
||||
func (t *Teller) askAgain(a asks.Ask, msgID string) error {
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
@@ -241,16 +236,6 @@ func (b *bed) askedOf(subject string) []published {
|
||||
}
|
||||
|
||||
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
|
||||
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
|
||||
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
|
||||
var said []string
|
||||
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
|
||||
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
|
||||
}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
|
||||
t.Errorf("an essential bucket refused: %v, want its grant named", err)
|
||||
}
|
||||
})
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.gate[key] = value
|
||||
@@ -533,8 +518,3 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R
|
||||
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// refusingBus refuses every bucket with err (issue 1945).
|
||||
type refusingBus struct{ err error }
|
||||
|
||||
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -26,8 +25,7 @@ func (t *Teller) OnViewAnswer(key string) {
|
||||
t.fillFromView(askKey)
|
||||
}
|
||||
|
||||
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now, and an
|
||||
// ask not answered yet is raised on the operator channel (issue 1906; assignerasks.go).
|
||||
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now.
|
||||
func (t *Teller) OnAssignerAsk(key string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
@@ -36,14 +34,10 @@ func (t *Teller) OnAssignerAsk(key string) {
|
||||
return
|
||||
}
|
||||
t.fillFromView(key)
|
||||
t.raiseAssignerAsk(key)
|
||||
}
|
||||
|
||||
// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the
|
||||
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet. A hold ask answered
|
||||
// already takes a later answer of the view that chose another option: the operator changed their mind (issue 1906 note 23: a
|
||||
// "keep" given on the phone could not be undone). Later means the view's answer was written after the answer that stands, so a
|
||||
// view answer read again at a start never replaces a newer answer from the phone.
|
||||
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet.
|
||||
func (t *Teller) fillFromView(askKey string) {
|
||||
if t.Assigner == nil || t.View == nil {
|
||||
return
|
||||
@@ -53,7 +47,7 @@ func (t *Teller) fillFromView(askKey string) {
|
||||
t.logf("%s is not answered from the view: %v", askKey, err)
|
||||
return
|
||||
}
|
||||
rawAnswer, at, found, err := t.viewAnswer(viewKey)
|
||||
rawAnswer, found, err := t.View.Get(viewKey)
|
||||
if err != nil {
|
||||
t.logf("%s: the view's answer could not be read, so the ask is left: %v", askKey, err)
|
||||
return
|
||||
@@ -89,14 +83,14 @@ func (t *Teller) fillFromView(askKey string) {
|
||||
t.logf("%s: the answer could not be read: %v", askKey, err)
|
||||
return
|
||||
}
|
||||
if has && answeredAsking(cur) == ask.Ask.ID && !t.changesMind(askKey, cur, va, at) {
|
||||
if has && answeredAsking(cur) == ask.Ask.ID {
|
||||
return // this asking is answered, from the view or elsewhere
|
||||
}
|
||||
words := "answered in the view (mesh-delivery-view): " + va.Option
|
||||
if w := strings.TrimSpace(va.Words); w != "" {
|
||||
words += ": " + w
|
||||
}
|
||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + at.UTC().Format(time.RFC3339Nano), Words: words})
|
||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + t.now().UTC().Format("2006-01-02T15:04:05.999999999Z07:00"), Words: words})
|
||||
if err != nil {
|
||||
t.logf("%s: the answer could not be made: %v", askKey, err)
|
||||
return
|
||||
@@ -109,48 +103,5 @@ func (t *Teller) fillFromView(askKey string) {
|
||||
t.logf("%s: another writer answered first, and the view's answer is not written", askKey)
|
||||
default:
|
||||
t.logf("%s: %s", askKey, words)
|
||||
t.cancelOpen(askKey, ask.Ask.ID) // answered in the view: the same ask on the phone is taken back
|
||||
}
|
||||
}
|
||||
|
||||
// timedAnswers is the view's bucket as the program reads it: each answer with the time it was written.
|
||||
type timedAnswers interface {
|
||||
GetAt(key string) (value []byte, at time.Time, found bool, err error)
|
||||
}
|
||||
|
||||
// viewAnswer is the view's answer at key and when it was written; a bucket that keeps no time gives the teller's now.
|
||||
func (t *Teller) viewAnswer(key string) ([]byte, time.Time, bool, error) {
|
||||
if tv, ok := t.View.(timedAnswers); ok {
|
||||
v, at, found, err := tv.GetAt(key)
|
||||
if err == nil && !at.IsZero() {
|
||||
return v, at, found, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, time.Time{}, false, err
|
||||
}
|
||||
return v, t.now(), found, nil
|
||||
}
|
||||
v, found, err := t.View.Get(key)
|
||||
return v, t.now(), found, err
|
||||
}
|
||||
|
||||
// changesMind says whether the view's answer va, written at at, replaces the answer cur of the same asking of the hold ask at
|
||||
// askKey: it chose another option and was written after cur's warrant.
|
||||
func (t *Teller) changesMind(askKey string, cur []byte, va delivery.ViewAnswer, at time.Time) bool {
|
||||
k, err := delivery.ParseAnyAskKey(askKey)
|
||||
if err != nil || k.Kind != delivery.AskKindHold {
|
||||
return false
|
||||
}
|
||||
a, err := delivery.DecodeAnswer(cur)
|
||||
if err != nil || a.Option == va.Option {
|
||||
return false
|
||||
}
|
||||
_, when, _ := strings.Cut(a.Warrant, "@")
|
||||
before, err := time.Parse(time.RFC3339Nano, when)
|
||||
if err != nil || !at.After(before) {
|
||||
t.logf("%s: the view's answer %s is not newer than the answer %s that stands (%s), and is not taken", askKey, va.Option, a.Option, when)
|
||||
return false
|
||||
}
|
||||
t.logf("%s: the operator changed the answer from %s to %s in the view", askKey, a.Option, va.Option)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -11,9 +10,8 @@ import (
|
||||
|
||||
const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
|
||||
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, shaped as the assigner composes it
|
||||
// (mesh-delivery-assigner internal/assign asks.go HoldAsk: two options of level acknowledge that bind no act, so the router
|
||||
// takes it; the value is `{"v":1,"ask":<the router's ask>}`).
|
||||
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, as a fixture (in release 5 the assigner writes
|
||||
// none yet; the value is `{"v":1,"ask":<the router's ask>}`).
|
||||
func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
||||
t.Helper()
|
||||
key, err := delivery.HoldAskKey("plex", fpHeld)
|
||||
@@ -22,7 +20,7 @@ func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
||||
}
|
||||
ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator,
|
||||
Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held",
|
||||
Options: []asks.Option{{ID: "keep", Label: "Keep it held", Does: "keeps it held", Level: asks.Acknowledge}, {ID: "release", Label: "Release it", Does: "lets the move go on", Level: asks.Acknowledge}}}
|
||||
Options: []asks.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: asks.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: asks.Approve}}}
|
||||
value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -119,11 +117,8 @@ func TestAViewAnswerOfAnOptionTheAskDoesNotOfferFillsNothing(t *testing.T) { //
|
||||
}
|
||||
}
|
||||
|
||||
// A view answer written before the answer that stands never replaces it (1906/N23: only a later one may, for a hold). The
|
||||
// view's time is fixed here, not the package's clock, which the integration tests set to the wall clock.
|
||||
func TestAViewAnswerNeverReplacesTheAnswerOfTheSameAsking(t *testing.T) { // 1297/B10
|
||||
b := newBed(t)
|
||||
b.t.Now = func() time.Time { return time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }
|
||||
assigner, view := withView(b)
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
@@ -179,53 +174,3 @@ func TestAStaleViewAnswerOfAnotherAskingFillsNothing(t *testing.T) { // 1297/B10
|
||||
t.Errorf("writes %d; log:\n%s", b.answers.writes, b.logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// timedView is the view's bucket with the time each answer was written.
|
||||
type timedView struct {
|
||||
memGate
|
||||
at map[string]time.Time
|
||||
}
|
||||
|
||||
func (v timedView) GetAt(key string) ([]byte, time.Time, bool, error) {
|
||||
b, ok := v.memGate[key]
|
||||
return b, v.at[key], ok, nil
|
||||
}
|
||||
|
||||
// 1906/N23-AC4: given a hold ask answered "keep" on the phone, when the operator answers "release" in the view later, then the
|
||||
// teller replaces the answer of that asking with "release", whose warrant names the asking and the view's time; a view answer
|
||||
// written before the phone's answer, the same option, or an ask of another kind replaces nothing.
|
||||
func TestALaterViewAnswerChangesAHoldsAnswer(t *testing.T) {
|
||||
setup := func(viewAt time.Time, option string) (*bed, string, asks.Ask) {
|
||||
b := newBed(t)
|
||||
assigner := memGate{}
|
||||
view := timedView{memGate: memGate{}, at: map[string]time.Time{}}
|
||||
b.t.Assigner, b.t.View = assigner, view
|
||||
key, value, ask := holdAsk(t)
|
||||
assigner[key] = value
|
||||
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + now.Add(time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator, via telegram, chose keep"})
|
||||
b.answers.written[key], b.answers.revs[key] = kept, 3
|
||||
view.memGate[key+".view"] = viewAnswer(t, option, "it was a flake")
|
||||
view.at[key+".view"] = viewAt
|
||||
b.t.OnViewAnswer(key + ".view")
|
||||
return b, key, ask
|
||||
}
|
||||
later := now.Add(2 * time.Hour)
|
||||
b, key, ask := setup(later, "release")
|
||||
a, err := delivery.DecodeAnswer(b.answers.written[key])
|
||||
if err != nil || b.answers.writes != 1 || a.Option != "release" || a.Warrant != ask.ID+"@"+later.UTC().Format(time.RFC3339Nano) {
|
||||
t.Fatalf("writes %d, answer %+v, %v; log:\n%s", b.answers.writes, a, err, b.logs.String())
|
||||
}
|
||||
for name, b := range map[string]*bed{
|
||||
"older": func() *bed { b, _, _ := setup(now, "release"); return b }(),
|
||||
"same": func() *bed { b, _, _ := setup(later, "keep"); return b }(),
|
||||
} {
|
||||
if b.answers.writes != 0 {
|
||||
t.Errorf("%s: the answer was replaced", name)
|
||||
}
|
||||
}
|
||||
// A restore ask answered stays answered: only a hold's answer may be changed.
|
||||
bb := newBed(t)
|
||||
if bb.t.changesMind("restore.ace.plex.2222222222222222222222222222222222222222222222222222222222222222", b.answers.written[key], delivery.ViewAnswer{Option: "keep"}, later.Add(time.Hour)) {
|
||||
t.Error("a restore ask's answer was changed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user